mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 21:17:56 +09:00
Compare commits
30
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ddc4460065 | ||
|
|
b0f3d15112 | ||
|
|
753499db01 | ||
|
|
1b6e682860 | ||
|
|
9baba12280 | ||
|
|
24b7b75ab2 | ||
|
|
0b59b7abfc | ||
|
|
942cbb4f66 | ||
|
|
96383a60d4 | ||
|
|
0cf14be531 | ||
|
|
95edd82555 | ||
|
|
6e02ca3056 | ||
|
|
966e8cc963 | ||
|
|
04b1c8fd3e | ||
|
|
f47b126c08 | ||
|
|
4438293b6d | ||
|
|
de4e92d9bf | ||
|
|
e7f88b60d0 | ||
|
|
fbb1cd7299 | ||
|
|
77820a44c9 | ||
|
|
e92ddd8609 | ||
|
|
a2bf2230ab | ||
|
|
75c6879b7e | ||
|
|
f51ef68179 | ||
|
|
8a5420f5f8 | ||
|
|
c9b7f1bb1a | ||
|
|
d9da1dd360 | ||
|
|
a8fa6b5f46 | ||
|
|
ebe0121938 | ||
|
|
01db9a764a |
No files matched your search
@@ -415,8 +415,9 @@ jobs:
|
||||
run: |
|
||||
echo "🔍 Checking if base image needs rebuild..."
|
||||
|
||||
CURRENT_HASH=$(sha256sum poetry.lock | cut -d' ' -f1)
|
||||
echo "Current poetry.lock hash: $CURRENT_HASH"
|
||||
# Hash both poetry.lock AND Dockerfile.base - rebuild if either changes
|
||||
CURRENT_HASH=$(cat poetry.lock backend/Dockerfile.base | sha256sum | cut -d' ' -f1)
|
||||
echo "Current combined hash (poetry.lock + Dockerfile.base): $CURRENT_HASH"
|
||||
|
||||
gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
|
||||
|
||||
@@ -424,14 +425,18 @@ jobs:
|
||||
EXISTING_HASH=""
|
||||
|
||||
if docker pull $BASE_IMAGE 2>/dev/null; then
|
||||
EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "poetry.lock.hash"}}' 2>/dev/null || echo "")
|
||||
EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "base.content.hash"}}' 2>/dev/null || echo "")
|
||||
# Fallback to old label name for backwards compatibility
|
||||
if [ -z "$EXISTING_HASH" ]; then
|
||||
EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "poetry.lock.hash"}}' 2>/dev/null || echo "")
|
||||
fi
|
||||
echo "Existing base image hash: $EXISTING_HASH"
|
||||
else
|
||||
echo "⚠️ No existing base image found"
|
||||
fi
|
||||
|
||||
if [ "$CURRENT_HASH" != "$EXISTING_HASH" ]; then
|
||||
echo "🔄 Base image needs rebuild (poetry.lock changed)"
|
||||
echo "🔄 Base image needs rebuild (poetry.lock or Dockerfile.base changed)"
|
||||
echo "needs_rebuild=true" >> $GITHUB_OUTPUT
|
||||
else
|
||||
echo "✅ Base image is up to date"
|
||||
|
||||
+6
-2
@@ -11,7 +11,9 @@
|
||||
# gcloud builds submit --config=cloudbuild-base.yaml --project=nomadkaraoke
|
||||
|
||||
# Use pre-built base image with all dependencies
|
||||
FROM us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest
|
||||
# Override with --build-arg BASE_IMAGE=karaoke-backend-base:local for local testing
|
||||
ARG BASE_IMAGE=us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
# Set working directory (should already exist from base)
|
||||
WORKDIR /app
|
||||
@@ -34,5 +36,7 @@ ENV PORT=8080
|
||||
# Expose port
|
||||
EXPOSE 8080
|
||||
|
||||
# Run the application
|
||||
# Run uvicorn directly
|
||||
# Note: Transmission daemon removed - torrent downloads now handled by
|
||||
# dedicated flacfetch VM service (see docs/00-current-plan/TORRENT-SERVICE-PLAN.md)
|
||||
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||
@@ -7,12 +7,15 @@
|
||||
FROM python:3.11-slim
|
||||
|
||||
# Install system dependencies required by karaoke-gen
|
||||
# Note: Transmission daemon removed - torrent downloads now handled by
|
||||
# dedicated flacfetch VM service (see docs/00-current-plan/TORRENT-SERVICE-PLAN.md)
|
||||
RUN apt-get update && apt-get install -y \
|
||||
ffmpeg \
|
||||
libsndfile1 \
|
||||
libsox-dev \
|
||||
sox \
|
||||
build-essential \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Set working directory
|
||||
@@ -34,9 +37,9 @@ RUN python -m spacy download en_core_web_sm
|
||||
# Label with build info
|
||||
ARG BUILD_DATE
|
||||
ARG BUILD_ID
|
||||
ARG POETRY_LOCK_HASH
|
||||
ARG BASE_CONTENT_HASH
|
||||
LABEL org.opencontainers.image.created="${BUILD_DATE}"
|
||||
LABEL org.opencontainers.image.revision="${BUILD_ID}"
|
||||
LABEL org.opencontainers.image.title="karaoke-backend-base"
|
||||
LABEL org.opencontainers.image.description="Base image with Python dependencies for karaoke-backend"
|
||||
LABEL poetry.lock.hash="${POETRY_LOCK_HASH}"
|
||||
LABEL org.opencontainers.image.description="Base image with Python dependencies and system tools for karaoke-backend"
|
||||
LABEL base.content.hash="${BASE_CONTENT_HASH}"
|
||||
@@ -82,15 +82,40 @@ class AudioSearchRequest(BaseModel):
|
||||
|
||||
|
||||
class AudioSearchResultResponse(BaseModel):
|
||||
"""A single audio search result."""
|
||||
"""A single audio search result.
|
||||
|
||||
Contains all fields needed for rich display using flacfetch's
|
||||
shared formatting functions.
|
||||
"""
|
||||
index: int
|
||||
title: str
|
||||
artist: str
|
||||
provider: str
|
||||
url: str
|
||||
duration: Optional[int] = None
|
||||
quality: Optional[str] = None
|
||||
source_id: Optional[str] = None
|
||||
provider: str # Maps to source_name in Release
|
||||
url: str # Maps to download_url in Release
|
||||
duration: Optional[int] = None # Maps to duration_seconds in Release
|
||||
quality: Optional[str] = None # Stringified quality
|
||||
source_id: Optional[str] = None # Maps to info_hash in Release
|
||||
seeders: Optional[int] = None
|
||||
target_file: Optional[str] = None
|
||||
# Additional fields for rich display (from Release.to_dict())
|
||||
year: Optional[int] = None
|
||||
label: Optional[str] = None
|
||||
edition_info: Optional[str] = None
|
||||
release_type: Optional[str] = None
|
||||
channel: Optional[str] = None # For YouTube
|
||||
view_count: Optional[int] = None # For YouTube
|
||||
size_bytes: Optional[int] = None
|
||||
target_file_size: Optional[int] = None
|
||||
track_pattern: Optional[str] = None
|
||||
match_score: Optional[float] = None
|
||||
# Pre-computed display fields
|
||||
formatted_size: Optional[str] = None
|
||||
formatted_duration: Optional[str] = None
|
||||
formatted_views: Optional[str] = None
|
||||
is_lossless: Optional[bool] = None
|
||||
quality_str: Optional[str] = None
|
||||
# Full quality object for Release.from_dict()
|
||||
quality_data: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class AudioSearchResponse(BaseModel):
|
||||
@@ -418,20 +443,50 @@ async def search_audio(
|
||||
message=f"Found {len(search_results)} audio sources. Waiting for selection."
|
||||
)
|
||||
|
||||
# Convert to response format
|
||||
result_responses = [
|
||||
AudioSearchResultResponse(
|
||||
index=r.index,
|
||||
title=r.title,
|
||||
artist=r.artist,
|
||||
provider=r.provider,
|
||||
url=r.url,
|
||||
duration=r.duration,
|
||||
quality=r.quality,
|
||||
source_id=r.source_id,
|
||||
# Convert to response format with full Release data for rich display
|
||||
result_responses = []
|
||||
for r in search_results:
|
||||
# Get full serialized data from raw_result if available
|
||||
raw_dict = {}
|
||||
if r.raw_result:
|
||||
try:
|
||||
raw_dict = r.raw_result.to_dict()
|
||||
except AttributeError:
|
||||
pass # Not a Release object
|
||||
|
||||
result_responses.append(
|
||||
AudioSearchResultResponse(
|
||||
index=r.index,
|
||||
title=r.title,
|
||||
artist=r.artist,
|
||||
provider=r.provider,
|
||||
url=r.url,
|
||||
duration=r.duration,
|
||||
quality=r.quality,
|
||||
source_id=r.source_id,
|
||||
seeders=raw_dict.get('seeders') or r.seeders,
|
||||
target_file=raw_dict.get('target_file') or r.target_file,
|
||||
# Additional fields for rich display
|
||||
year=raw_dict.get('year'),
|
||||
label=raw_dict.get('label'),
|
||||
edition_info=raw_dict.get('edition_info'),
|
||||
release_type=raw_dict.get('release_type'),
|
||||
channel=raw_dict.get('channel'),
|
||||
view_count=raw_dict.get('view_count'),
|
||||
size_bytes=raw_dict.get('size_bytes'),
|
||||
target_file_size=raw_dict.get('target_file_size'),
|
||||
track_pattern=raw_dict.get('track_pattern'),
|
||||
match_score=raw_dict.get('match_score'),
|
||||
# Pre-computed display fields
|
||||
formatted_size=raw_dict.get('formatted_size'),
|
||||
formatted_duration=raw_dict.get('formatted_duration'),
|
||||
formatted_views=raw_dict.get('formatted_views'),
|
||||
is_lossless=raw_dict.get('is_lossless'),
|
||||
quality_str=raw_dict.get('quality_str'),
|
||||
# Full quality object for Release.from_dict()
|
||||
quality_data=raw_dict.get('quality'),
|
||||
)
|
||||
)
|
||||
for r in search_results
|
||||
]
|
||||
|
||||
return AudioSearchResponse(
|
||||
status="awaiting_selection",
|
||||
@@ -473,6 +528,8 @@ async def get_audio_search_results(job_id: str):
|
||||
"status": "success",
|
||||
"job_id": job_id,
|
||||
"job_status": job.status,
|
||||
"artist": job.audio_search_artist or job.artist,
|
||||
"title": job.audio_search_title or job.title,
|
||||
"results": search_results,
|
||||
"results_count": len(search_results),
|
||||
}
|
||||
|
||||
@@ -1,10 +1,64 @@
|
||||
"""
|
||||
Health check routes.
|
||||
"""
|
||||
import os
|
||||
import logging
|
||||
from fastapi import APIRouter
|
||||
from typing import Dict
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from backend.version import VERSION
|
||||
from backend.services.flacfetch_client import get_flacfetch_client
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def check_transmission_status() -> Dict[str, Any]:
|
||||
"""Check if Transmission daemon is available and responsive."""
|
||||
try:
|
||||
import transmission_rpc
|
||||
host = os.environ.get("TRANSMISSION_HOST", "localhost")
|
||||
port = int(os.environ.get("TRANSMISSION_PORT", "9091"))
|
||||
|
||||
client = transmission_rpc.Client(host=host, port=port, timeout=5)
|
||||
stats = client.session_stats()
|
||||
|
||||
# Get detailed torrent info
|
||||
torrents = client.get_torrents()
|
||||
torrent_details = []
|
||||
for t in torrents:
|
||||
torrent_info = {
|
||||
"name": t.name,
|
||||
"progress": round(t.progress, 1),
|
||||
"status": str(t.status),
|
||||
"peers": t.peers_connected if hasattr(t, 'peers_connected') else 0,
|
||||
"download_speed": round(t.rate_download / 1024, 1) if hasattr(t, 'rate_download') else 0, # KB/s
|
||||
}
|
||||
# Check if stalled (downloading but no progress and no peers)
|
||||
if t.status.downloading and t.progress < 100 and torrent_info['peers'] == 0:
|
||||
torrent_info['stalled'] = True
|
||||
torrent_details.append(torrent_info)
|
||||
|
||||
return {
|
||||
"available": True,
|
||||
"host": host,
|
||||
"port": port,
|
||||
"download_dir": stats.download_dir if hasattr(stats, 'download_dir') else None,
|
||||
"active_torrent_count": stats.active_torrent_count if hasattr(stats, 'active_torrent_count') else 0,
|
||||
"torrents": torrent_details,
|
||||
}
|
||||
except ImportError as e:
|
||||
return {
|
||||
"available": False,
|
||||
"error": f"transmission_rpc not installed: {e}",
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"available": False,
|
||||
"host": os.environ.get("TRANSMISSION_HOST", "localhost"),
|
||||
"port": int(os.environ.get("TRANSMISSION_PORT", "9091")),
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
@@ -16,6 +70,67 @@ async def health_check() -> Dict[str, str]:
|
||||
}
|
||||
|
||||
|
||||
async def check_flacfetch_service_status() -> Dict[str, Any]:
|
||||
"""Check if remote flacfetch service is available and healthy."""
|
||||
client = get_flacfetch_client()
|
||||
|
||||
if not client:
|
||||
return {
|
||||
"configured": False,
|
||||
"message": "Remote flacfetch service not configured (FLACFETCH_API_URL not set)",
|
||||
}
|
||||
|
||||
try:
|
||||
health = await client.health_check()
|
||||
return {
|
||||
"configured": True,
|
||||
"available": True,
|
||||
"status": health.get("status"),
|
||||
"version": health.get("version"),
|
||||
"transmission": health.get("transmission", {}),
|
||||
"disk": health.get("disk", {}),
|
||||
"providers": health.get("providers", {}),
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"configured": True,
|
||||
"available": False,
|
||||
"error": str(e),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/health/detailed")
|
||||
async def detailed_health_check() -> Dict[str, Any]:
|
||||
"""
|
||||
Detailed health check including dependencies.
|
||||
|
||||
Use this to debug issues with Transmission, flacfetch service, etc.
|
||||
"""
|
||||
transmission_status = check_transmission_status()
|
||||
flacfetch_status = await check_flacfetch_service_status()
|
||||
|
||||
# Log for debugging
|
||||
if not transmission_status.get("available"):
|
||||
logger.warning(f"Local Transmission not available: {transmission_status.get('error')}")
|
||||
else:
|
||||
logger.info(f"Local Transmission available at {transmission_status.get('host')}:{transmission_status.get('port')}")
|
||||
|
||||
if flacfetch_status.get("configured") and not flacfetch_status.get("available"):
|
||||
logger.warning(f"Remote flacfetch service not available: {flacfetch_status.get('error')}")
|
||||
elif flacfetch_status.get("available"):
|
||||
logger.info(f"Remote flacfetch service healthy: {flacfetch_status.get('status')}")
|
||||
|
||||
return {
|
||||
"status": "healthy",
|
||||
"service": "karaoke-gen-backend",
|
||||
"version": VERSION,
|
||||
"dependencies": {
|
||||
"transmission_local": transmission_status,
|
||||
"flacfetch_remote": flacfetch_status,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@router.get("/readiness")
|
||||
async def readiness_check() -> Dict[str, str]:
|
||||
"""Readiness check endpoint for Cloud Run."""
|
||||
|
||||
@@ -56,6 +56,12 @@ class Settings(BaseSettings):
|
||||
# Storage paths
|
||||
temp_dir: str = os.getenv("TEMP_DIR", "/tmp/karaoke-gen")
|
||||
|
||||
# Flacfetch remote service (for torrent downloads)
|
||||
# When configured, audio search uses the remote flacfetch HTTP API instead of local flacfetch.
|
||||
# This is required for torrent downloads since Cloud Run doesn't support BitTorrent.
|
||||
flacfetch_api_url: Optional[str] = os.getenv("FLACFETCH_API_URL") # e.g., http://10.0.0.5:8080
|
||||
flacfetch_api_key: Optional[str] = os.getenv("FLACFETCH_API_KEY")
|
||||
|
||||
# Default distribution settings (can be overridden per-request)
|
||||
default_dropbox_path: Optional[str] = os.getenv("DEFAULT_DROPBOX_PATH")
|
||||
default_gdrive_folder_id: Optional[str] = os.getenv("DEFAULT_GDRIVE_FOLDER_ID")
|
||||
|
||||
@@ -1,79 +1,67 @@
|
||||
"""
|
||||
Audio search service for finding and downloading audio files.
|
||||
|
||||
This service integrates with flacfetch to search for audio from various sources
|
||||
(YouTube, music trackers, etc.) and download the selected audio file.
|
||||
This service integrates with karaoke_gen.audio_fetcher (which wraps flacfetch)
|
||||
to search for audio from various sources (YouTube, music trackers, etc.)
|
||||
and download the selected audio file.
|
||||
|
||||
This is a thin wrapper that adds backend-specific functionality:
|
||||
- Caching raw results for API-based selection flow
|
||||
- Firestore-compatible serialization
|
||||
- Singleton pattern for service lifecycle
|
||||
- Remote flacfetch service integration for torrent downloads
|
||||
|
||||
When FLACFETCH_API_URL is configured, this service uses the remote flacfetch
|
||||
HTTP API for search and download operations. This is required for torrent
|
||||
downloads since Cloud Run doesn't support BitTorrent peer connections.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from dataclasses import dataclass, asdict
|
||||
from typing import List, Optional, Dict, Any
|
||||
from typing import List, Optional
|
||||
|
||||
# Import shared classes from karaoke_gen.audio_fetcher - single source of truth
|
||||
# Note: Import directly from audio_fetcher module, not karaoke_gen.__init__
|
||||
# to avoid pulling in heavier dependencies like lyrics_transcriber
|
||||
from karaoke_gen.audio_fetcher import (
|
||||
FlacFetcher,
|
||||
AudioSearchResult,
|
||||
AudioFetchResult,
|
||||
AudioFetcherError,
|
||||
NoResultsError,
|
||||
DownloadError,
|
||||
)
|
||||
|
||||
from .flacfetch_client import get_flacfetch_client, FlacfetchServiceError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioSearchResult:
|
||||
"""
|
||||
Represents a single search result for audio.
|
||||
|
||||
This is a serializable representation of a search result that can be
|
||||
stored in Firestore and sent to clients.
|
||||
"""
|
||||
title: str
|
||||
artist: str
|
||||
provider: str
|
||||
url: str
|
||||
duration: Optional[int] = None # Duration in seconds
|
||||
quality: Optional[str] = None # e.g., "FLAC", "320kbps", etc.
|
||||
source_id: Optional[str] = None # Unique ID from the source
|
||||
index: int = 0 # Index in the results list (for selection)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dict for JSON serialization."""
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AudioSearchResult":
|
||||
"""Create from dict."""
|
||||
return cls(**data)
|
||||
# Re-export exception under backend's naming convention for compatibility
|
||||
AudioSearchError = AudioFetcherError
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioDownloadResult:
|
||||
"""Result of downloading an audio file."""
|
||||
filepath: str
|
||||
artist: str
|
||||
title: str
|
||||
provider: str
|
||||
duration: Optional[int] = None
|
||||
quality: Optional[str] = None
|
||||
|
||||
|
||||
class AudioSearchError(Exception):
|
||||
"""Base exception for audio search errors."""
|
||||
pass
|
||||
|
||||
|
||||
class NoResultsError(AudioSearchError):
|
||||
"""Raised when no search results are found."""
|
||||
pass
|
||||
|
||||
|
||||
class DownloadError(AudioSearchError):
|
||||
"""Raised when download fails."""
|
||||
pass
|
||||
# Also alias AudioFetchResult as AudioDownloadResult for backwards compatibility
|
||||
AudioDownloadResult = AudioFetchResult
|
||||
|
||||
|
||||
class AudioSearchService:
|
||||
"""
|
||||
Service for searching and downloading audio files using flacfetch.
|
||||
Service for searching and downloading audio files.
|
||||
|
||||
This service is used by the backend to:
|
||||
1. Search for audio by artist/title
|
||||
2. Return results for user selection
|
||||
3. Download the selected audio to a local file
|
||||
This is a thin wrapper around karaoke_gen.audio_fetcher.FlacFetcher that adds
|
||||
backend-specific functionality:
|
||||
- Caches raw results for API-based selection (client picks by index)
|
||||
- Provides Firestore-compatible serialization
|
||||
- Manages service lifecycle as singleton
|
||||
- Remote flacfetch service integration for torrent downloads
|
||||
|
||||
When FLACFETCH_API_URL is configured, this service uses the remote flacfetch
|
||||
HTTP API for search and download operations. This enables torrent downloads
|
||||
which are not possible in Cloud Run due to network restrictions.
|
||||
|
||||
The actual flacfetch integration is in karaoke_gen.audio_fetcher.FlacFetcher,
|
||||
which is shared between local CLI and cloud backend.
|
||||
"""
|
||||
|
||||
# Sentinel value to indicate "use environment variable"
|
||||
@@ -93,60 +81,42 @@ class AudioSearchService:
|
||||
"""
|
||||
# Use environment variables if not explicitly provided
|
||||
if redacted_api_key is self._USE_ENV:
|
||||
self._redacted_api_key = os.environ.get("REDACTED_API_KEY")
|
||||
else:
|
||||
self._redacted_api_key = redacted_api_key
|
||||
redacted_api_key = os.environ.get("REDACTED_API_KEY")
|
||||
|
||||
if ops_api_key is self._USE_ENV:
|
||||
self._ops_api_key = os.environ.get("OPS_API_KEY")
|
||||
else:
|
||||
self._ops_api_key = ops_api_key
|
||||
|
||||
self._manager = None
|
||||
self._raw_results = {} # Cache raw results for download
|
||||
|
||||
def _get_manager(self):
|
||||
"""Lazily initialize and return the FetchManager."""
|
||||
if self._manager is None:
|
||||
try:
|
||||
from flacfetch.core.manager import FetchManager
|
||||
from flacfetch.providers.youtube import YouTubeProvider
|
||||
except ImportError as e:
|
||||
logger.error(f"flacfetch not installed: {e}")
|
||||
raise AudioSearchError(
|
||||
"flacfetch library is not installed. "
|
||||
"Install with: pip install flacfetch"
|
||||
) from e
|
||||
|
||||
self._manager = FetchManager()
|
||||
|
||||
# Add providers based on available API keys
|
||||
if self._redacted_api_key:
|
||||
try:
|
||||
from flacfetch.providers.redacted import RedactedProvider
|
||||
self._manager.add_provider(RedactedProvider(api_key=self._redacted_api_key))
|
||||
logger.debug("Added Redacted provider")
|
||||
except ImportError:
|
||||
logger.warning("Redacted provider not available")
|
||||
|
||||
if self._ops_api_key:
|
||||
try:
|
||||
from flacfetch.providers.ops import OPSProvider
|
||||
self._manager.add_provider(OPSProvider(api_key=self._ops_api_key))
|
||||
logger.debug("Added OPS provider")
|
||||
except ImportError:
|
||||
logger.warning("OPS provider not available")
|
||||
|
||||
# Always add YouTube as a fallback provider
|
||||
self._manager.add_provider(YouTubeProvider())
|
||||
logger.debug("Added YouTube provider")
|
||||
ops_api_key = os.environ.get("OPS_API_KEY")
|
||||
|
||||
return self._manager
|
||||
# Check for remote flacfetch client
|
||||
self._remote_client = get_flacfetch_client()
|
||||
|
||||
# Log which mode we're using
|
||||
if self._remote_client:
|
||||
logger.info("AudioSearchService using REMOTE flacfetch service (torrent downloads enabled)")
|
||||
else:
|
||||
logger.info("AudioSearchService using LOCAL flacfetch (YouTube only in Cloud Run)")
|
||||
|
||||
# Delegate to shared FlacFetcher implementation (for local mode or fallback)
|
||||
self._fetcher = FlacFetcher(
|
||||
redacted_api_key=redacted_api_key,
|
||||
ops_api_key=ops_api_key,
|
||||
)
|
||||
|
||||
# Cache search results for API-based selection flow
|
||||
# Key: index, Value: AudioSearchResult (with raw_result)
|
||||
self._cached_results: List[AudioSearchResult] = []
|
||||
|
||||
# Cache for remote search results (search_id -> results)
|
||||
self._remote_search_id: Optional[str] = None
|
||||
|
||||
def search(self, artist: str, title: str) -> List[AudioSearchResult]:
|
||||
"""
|
||||
Search for audio matching the given artist and title.
|
||||
|
||||
Results are cached internally for later download via download().
|
||||
|
||||
If a remote flacfetch service is configured (FLACFETCH_API_URL), uses that
|
||||
for better torrent download support. Otherwise falls back to local flacfetch.
|
||||
|
||||
Args:
|
||||
artist: The artist name to search for
|
||||
title: The track title to search for
|
||||
@@ -156,49 +126,90 @@ class AudioSearchService:
|
||||
|
||||
Raises:
|
||||
NoResultsError: If no results are found
|
||||
AudioSearchError: For other errors
|
||||
AudioSearchError: For other errors (e.g., flacfetch not installed)
|
||||
"""
|
||||
try:
|
||||
from flacfetch.core.models import TrackQuery
|
||||
except ImportError as e:
|
||||
raise AudioSearchError("flacfetch not installed") from e
|
||||
|
||||
manager = self._get_manager()
|
||||
query = TrackQuery(artist=artist, title=title)
|
||||
|
||||
logger.info(f"Searching for audio: {artist} - {title}")
|
||||
|
||||
try:
|
||||
results = manager.search(query)
|
||||
# Try remote flacfetch service first if configured
|
||||
if self._remote_client:
|
||||
return self._search_remote(artist, title)
|
||||
|
||||
# Fallback to local flacfetch
|
||||
results = self._fetcher.search(artist, title)
|
||||
|
||||
# Cache results for later download
|
||||
self._cached_results = results
|
||||
self._remote_search_id = None
|
||||
|
||||
logger.info(f"Found {len(results)} results for: {artist} - {title}")
|
||||
return results
|
||||
|
||||
except (NoResultsError, AudioFetcherError):
|
||||
# Re-raise these as-is
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Search failed: {e}")
|
||||
raise AudioSearchError(f"Search failed: {e}") from e
|
||||
|
||||
def _search_remote(self, artist: str, title: str) -> List[AudioSearchResult]:
|
||||
"""
|
||||
Search using the remote flacfetch service.
|
||||
|
||||
if not results:
|
||||
raise NoResultsError(f"No results found for: {artist} - {title}")
|
||||
|
||||
# Convert to our AudioSearchResult format and cache raw results
|
||||
search_results = []
|
||||
self._raw_results.clear()
|
||||
|
||||
for i, result in enumerate(results):
|
||||
search_result = AudioSearchResult(
|
||||
title=getattr(result, "title", title),
|
||||
artist=getattr(result, "artist", artist),
|
||||
provider=getattr(result, "provider", "Unknown"),
|
||||
url=getattr(result, "url", ""),
|
||||
duration=getattr(result, "duration", None),
|
||||
quality=getattr(result, "quality", None),
|
||||
source_id=getattr(result, "id", str(i)),
|
||||
index=i,
|
||||
)
|
||||
search_results.append(search_result)
|
||||
Runs async code in a sync context for compatibility with existing API.
|
||||
"""
|
||||
try:
|
||||
# Run async search in sync context
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
response = loop.run_until_complete(
|
||||
self._remote_client.search(artist, title)
|
||||
)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
# Cache raw result for later download
|
||||
self._raw_results[i] = result
|
||||
|
||||
logger.info(f"Found {len(search_results)} results for: {artist} - {title}")
|
||||
return search_results
|
||||
# Check for empty results
|
||||
if not response.get("results"):
|
||||
raise NoResultsError(f"No results found for: {artist} - {title}")
|
||||
|
||||
# Store search_id for download
|
||||
self._remote_search_id = response.get("search_id")
|
||||
|
||||
# Convert remote results to AudioSearchResult objects
|
||||
results = []
|
||||
for item in response.get("results", []):
|
||||
result = AudioSearchResult(
|
||||
title=item.get("title", "Unknown"),
|
||||
artist=item.get("artist", "Unknown"),
|
||||
provider=item.get("provider", "Unknown"),
|
||||
quality=item.get("quality", ""),
|
||||
size_bytes=item.get("size_bytes"),
|
||||
duration_seconds=item.get("duration_seconds"),
|
||||
is_lossless=item.get("is_lossless", False),
|
||||
seeders=item.get("seeders"),
|
||||
year=item.get("year"),
|
||||
extra_info={
|
||||
"index": item.get("index"),
|
||||
"target_file": item.get("target_file"),
|
||||
"view_count": item.get("view_count"),
|
||||
"channel": item.get("channel"),
|
||||
"remote": True, # Flag for download logic
|
||||
},
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
# Cache results
|
||||
self._cached_results = results
|
||||
|
||||
logger.info(f"Found {len(results)} results from remote flacfetch for: {artist} - {title}")
|
||||
return results
|
||||
|
||||
except FlacfetchServiceError as e:
|
||||
logger.warning(f"Remote flacfetch search failed, falling back to local: {e}")
|
||||
# Fallback to local search
|
||||
self._remote_search_id = None
|
||||
results = self._fetcher.search(artist, title)
|
||||
self._cached_results = results
|
||||
return results
|
||||
|
||||
def select_best(self, results: List[AudioSearchResult]) -> int:
|
||||
"""
|
||||
@@ -212,90 +223,136 @@ class AudioSearchService:
|
||||
Returns:
|
||||
Index of the best result
|
||||
"""
|
||||
if not results:
|
||||
return 0
|
||||
|
||||
manager = self._get_manager()
|
||||
|
||||
# Get raw results for the indices we have
|
||||
raw_results = [self._raw_results.get(r.index) for r in results if r.index in self._raw_results]
|
||||
|
||||
if raw_results:
|
||||
try:
|
||||
best = manager.select_best(raw_results)
|
||||
# Find index of best result
|
||||
for i, raw in enumerate(raw_results):
|
||||
if raw == best:
|
||||
return results[i].index
|
||||
except Exception as e:
|
||||
logger.warning(f"select_best failed, using first result: {e}")
|
||||
|
||||
# Fallback: return first result
|
||||
return results[0].index if results else 0
|
||||
return self._fetcher.select_best(results)
|
||||
|
||||
def download(
|
||||
self,
|
||||
result_index: int,
|
||||
output_dir: str,
|
||||
output_filename: Optional[str] = None,
|
||||
gcs_path: Optional[str] = None,
|
||||
) -> AudioDownloadResult:
|
||||
"""
|
||||
Download audio from a search result.
|
||||
Download audio from a cached search result.
|
||||
|
||||
This method uses the cached results from the last search() call.
|
||||
The API flow is:
|
||||
1. Client calls search() -> gets list of results
|
||||
2. Client picks an index
|
||||
3. Client calls download(index) -> gets downloaded file
|
||||
|
||||
If a remote flacfetch service is configured and the search was performed
|
||||
remotely (for torrent sources), uses the remote service for download.
|
||||
|
||||
Args:
|
||||
result_index: Index of the result to download (from search results)
|
||||
output_dir: Directory to save the downloaded file
|
||||
output_filename: Optional filename (without extension)
|
||||
gcs_path: Optional GCS path for remote uploads (e.g., "uploads/job123/audio/")
|
||||
|
||||
Returns:
|
||||
AudioDownloadResult with the downloaded file path
|
||||
|
||||
Raises:
|
||||
DownloadError: If download fails
|
||||
DownloadError: If download fails or no cached result for index
|
||||
"""
|
||||
if result_index not in self._raw_results:
|
||||
raise DownloadError(f"No cached result for index {result_index}. Run search() first.")
|
||||
if result_index < 0 or result_index >= len(self._cached_results):
|
||||
raise DownloadError(
|
||||
f"No cached result for index {result_index}. "
|
||||
f"Available indices: 0-{len(self._cached_results) - 1}. "
|
||||
"Run search() first."
|
||||
)
|
||||
|
||||
raw_result = self._raw_results[result_index]
|
||||
manager = self._get_manager()
|
||||
result = self._cached_results[result_index]
|
||||
|
||||
# Ensure output directory exists
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
logger.info(f"Downloading: {result.artist} - {result.title} from {result.provider}")
|
||||
|
||||
# Get metadata from raw result
|
||||
artist = getattr(raw_result, "artist", "Unknown")
|
||||
title = getattr(raw_result, "title", "Unknown")
|
||||
provider = getattr(raw_result, "provider", "Unknown")
|
||||
# Check if this was a remote search (torrent sources need remote download)
|
||||
if self._remote_search_id and self._remote_client:
|
||||
# Check if this is a torrent source that needs remote handling
|
||||
is_remote = result.extra_info.get("remote", False) if result.extra_info else False
|
||||
is_torrent = result.provider in ["Redacted", "OPS"]
|
||||
|
||||
if is_remote or is_torrent:
|
||||
return self._download_remote(result_index, output_dir, output_filename, gcs_path)
|
||||
|
||||
# Generate filename if not provided
|
||||
if output_filename is None:
|
||||
output_filename = f"{artist} - {title}"
|
||||
# Delegate to shared FlacFetcher (local download)
|
||||
fetch_result = self._fetcher.download(result, output_dir, output_filename)
|
||||
|
||||
logger.info(f"Downloading: {artist} - {title} from {provider}")
|
||||
logger.info(f"Downloaded to: {fetch_result.filepath}")
|
||||
|
||||
return fetch_result
|
||||
|
||||
def _download_remote(
|
||||
self,
|
||||
result_index: int,
|
||||
output_dir: str,
|
||||
output_filename: Optional[str] = None,
|
||||
gcs_path: Optional[str] = None,
|
||||
) -> AudioDownloadResult:
|
||||
"""
|
||||
Download using the remote flacfetch service.
|
||||
|
||||
The remote service downloads via torrent and optionally uploads to GCS.
|
||||
"""
|
||||
if not self._remote_search_id:
|
||||
raise DownloadError("No remote search ID - run search() first")
|
||||
|
||||
result = self._cached_results[result_index]
|
||||
|
||||
try:
|
||||
filepath = manager.download(
|
||||
raw_result,
|
||||
output_path=output_dir,
|
||||
output_filename=output_filename,
|
||||
)
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
# Start download
|
||||
download_id = loop.run_until_complete(
|
||||
self._remote_client.download(
|
||||
search_id=self._remote_search_id,
|
||||
result_index=result_index,
|
||||
output_filename=output_filename,
|
||||
gcs_path=gcs_path,
|
||||
)
|
||||
)
|
||||
|
||||
logger.info(f"Remote download started: {download_id}")
|
||||
|
||||
# Wait for completion with progress logging
|
||||
def log_progress(status):
|
||||
progress = status.get("progress", 0)
|
||||
speed = status.get("download_speed_kbps", 0)
|
||||
logger.debug(f"Download progress: {progress:.1f}% ({speed:.1f} KB/s)")
|
||||
|
||||
final_status = loop.run_until_complete(
|
||||
self._remote_client.wait_for_download(
|
||||
download_id,
|
||||
timeout=600, # 10 minute timeout
|
||||
progress_callback=log_progress,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
if filepath is None:
|
||||
raise DownloadError(f"Download returned no file path for: {artist} - {title}")
|
||||
# Determine file path
|
||||
filepath = final_status.get("gcs_path") or final_status.get("output_path")
|
||||
|
||||
logger.info(f"Downloaded to: {filepath}")
|
||||
if not filepath:
|
||||
raise DownloadError("Remote download completed but no file path returned")
|
||||
|
||||
logger.info(f"Remote download complete: {filepath}")
|
||||
|
||||
return AudioDownloadResult(
|
||||
filepath=filepath,
|
||||
artist=artist,
|
||||
title=title,
|
||||
provider=provider,
|
||||
duration=getattr(raw_result, "duration", None),
|
||||
quality=getattr(raw_result, "quality", None),
|
||||
artist=result.artist,
|
||||
title=result.title,
|
||||
provider=result.provider,
|
||||
quality=result.quality,
|
||||
)
|
||||
|
||||
except FlacfetchServiceError as e:
|
||||
raise DownloadError(f"Remote download failed: {e}") from e
|
||||
except Exception as e:
|
||||
raise DownloadError(f"Failed to download {artist} - {title}: {e}") from e
|
||||
logger.error(f"Remote download error: {e}")
|
||||
raise DownloadError(f"Remote download failed: {e}") from e
|
||||
|
||||
def search_and_download_auto(
|
||||
self,
|
||||
@@ -303,6 +360,7 @@ class AudioSearchService:
|
||||
title: str,
|
||||
output_dir: str,
|
||||
output_filename: Optional[str] = None,
|
||||
gcs_path: Optional[str] = None,
|
||||
) -> AudioDownloadResult:
|
||||
"""
|
||||
Search for audio and automatically download the best result.
|
||||
@@ -315,6 +373,7 @@ class AudioSearchService:
|
||||
title: The track title to search for
|
||||
output_dir: Directory to save the downloaded file
|
||||
output_filename: Optional filename (without extension)
|
||||
gcs_path: Optional GCS path for remote uploads
|
||||
|
||||
Returns:
|
||||
AudioDownloadResult with the downloaded file path
|
||||
@@ -328,10 +387,64 @@ class AudioSearchService:
|
||||
|
||||
# Select best
|
||||
best_index = self.select_best(results)
|
||||
logger.info(f"Auto-selected result {best_index}: {results[best_index].artist} - {results[best_index].title} ({results[best_index].provider})")
|
||||
best_result = results[best_index]
|
||||
logger.info(
|
||||
f"Auto-selected result {best_index}: "
|
||||
f"{best_result.artist} - {best_result.title} ({best_result.provider})"
|
||||
)
|
||||
|
||||
# Download
|
||||
return self.download(best_index, output_dir, output_filename)
|
||||
return self.download(best_index, output_dir, output_filename, gcs_path)
|
||||
|
||||
async def search_async(self, artist: str, title: str) -> List[AudioSearchResult]:
|
||||
"""
|
||||
Async version of search for use in async routes.
|
||||
|
||||
Note: Currently wraps sync search in executor. Future optimization
|
||||
could make this fully async.
|
||||
"""
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(None, self.search, artist, title)
|
||||
|
||||
async def download_async(
|
||||
self,
|
||||
result_index: int,
|
||||
output_dir: str,
|
||||
output_filename: Optional[str] = None,
|
||||
gcs_path: Optional[str] = None,
|
||||
) -> AudioDownloadResult:
|
||||
"""
|
||||
Async version of download for use in async routes.
|
||||
|
||||
Note: Currently wraps sync download in executor. Future optimization
|
||||
could make this fully async.
|
||||
"""
|
||||
loop = asyncio.get_event_loop()
|
||||
return await loop.run_in_executor(
|
||||
None,
|
||||
lambda: self.download(result_index, output_dir, output_filename, gcs_path)
|
||||
)
|
||||
|
||||
def is_remote_enabled(self) -> bool:
|
||||
"""Check if remote flacfetch service is configured."""
|
||||
return self._remote_client is not None
|
||||
|
||||
async def check_remote_health(self) -> Optional[dict]:
|
||||
"""
|
||||
Check health of remote flacfetch service.
|
||||
|
||||
Returns:
|
||||
Health status dict if remote service is configured and healthy,
|
||||
None if not configured or unhealthy.
|
||||
"""
|
||||
if not self._remote_client:
|
||||
return None
|
||||
|
||||
try:
|
||||
return await self._remote_client.health_check()
|
||||
except Exception as e:
|
||||
logger.warning(f"Remote flacfetch health check failed: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# Singleton instance
|
||||
@@ -344,5 +457,3 @@ def get_audio_search_service() -> AudioSearchService:
|
||||
if _audio_search_service is None:
|
||||
_audio_search_service = AudioSearchService()
|
||||
return _audio_search_service
|
||||
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
"""
|
||||
Client for the remote flacfetch HTTP API service.
|
||||
|
||||
This client communicates with a dedicated flacfetch VM that handles:
|
||||
- BitTorrent downloads from private trackers (Redacted, OPS)
|
||||
- YouTube downloads
|
||||
- GCS uploads of downloaded files
|
||||
|
||||
The flacfetch service provides:
|
||||
- Full peer connectivity for torrents (not possible in Cloud Run)
|
||||
- Indefinite seeding of completed torrents
|
||||
- Automatic disk cleanup
|
||||
|
||||
Usage:
|
||||
client = get_flacfetch_client()
|
||||
if client:
|
||||
# Search for audio
|
||||
search_result = await client.search("ABBA", "Waterloo")
|
||||
|
||||
# Download the best result
|
||||
download_id = await client.download(
|
||||
search_id=search_result["search_id"],
|
||||
result_index=0,
|
||||
gcs_path="uploads/job123/audio/",
|
||||
)
|
||||
|
||||
# Wait for completion
|
||||
result = await client.wait_for_download(download_id)
|
||||
print(f"Downloaded to: {result['gcs_path']}")
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FlacfetchServiceError(Exception):
|
||||
"""Error communicating with or returned from flacfetch service."""
|
||||
pass
|
||||
|
||||
|
||||
class FlacfetchClient:
|
||||
"""
|
||||
Client for remote flacfetch HTTP API.
|
||||
|
||||
All endpoints require authentication via X-API-Key header.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
base_url: str,
|
||||
api_key: str,
|
||||
timeout: int = 60,
|
||||
):
|
||||
"""
|
||||
Initialize flacfetch client.
|
||||
|
||||
Args:
|
||||
base_url: Base URL of flacfetch service (e.g., http://10.0.0.5:8080)
|
||||
api_key: API key for authentication
|
||||
timeout: Default request timeout in seconds
|
||||
"""
|
||||
self.base_url = base_url.rstrip('/')
|
||||
self.api_key = api_key
|
||||
self.timeout = timeout
|
||||
logger.info(f"FlacfetchClient initialized with base_url={self.base_url}")
|
||||
|
||||
def _headers(self) -> Dict[str, str]:
|
||||
"""Get request headers with authentication."""
|
||||
return {
|
||||
"X-API-Key": self.api_key,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
async def health_check(self) -> Dict[str, Any]:
|
||||
"""
|
||||
Check if flacfetch service is healthy.
|
||||
|
||||
Returns:
|
||||
Health status dict with transmission, disk, and provider info
|
||||
|
||||
Raises:
|
||||
FlacfetchServiceError: If service is unhealthy or unreachable
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{self.base_url}/health",
|
||||
headers=self._headers(),
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
if data.get("status") not in ["healthy", "degraded"]:
|
||||
raise FlacfetchServiceError(f"Service unhealthy: {data}")
|
||||
|
||||
return data
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"Cannot reach flacfetch service: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise FlacfetchServiceError(f"Health check failed: {e.response.status_code}")
|
||||
|
||||
async def search(
|
||||
self,
|
||||
artist: str,
|
||||
title: str,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Search for audio matching artist and title.
|
||||
|
||||
Args:
|
||||
artist: Artist name
|
||||
title: Track title
|
||||
|
||||
Returns:
|
||||
Search response dict with search_id and results list
|
||||
|
||||
Raises:
|
||||
FlacfetchServiceError: On search failure
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(
|
||||
f"{self.base_url}/search",
|
||||
headers=self._headers(),
|
||||
json={"artist": artist, "title": title},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
|
||||
if resp.status_code == 404:
|
||||
# No results found - return empty results
|
||||
return {
|
||||
"search_id": None,
|
||||
"artist": artist,
|
||||
"title": title,
|
||||
"results": [],
|
||||
"results_count": 0,
|
||||
}
|
||||
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"Search request failed: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise FlacfetchServiceError(f"Search failed: {e.response.status_code} - {e.response.text}")
|
||||
|
||||
async def download(
|
||||
self,
|
||||
search_id: str,
|
||||
result_index: int,
|
||||
output_filename: Optional[str] = None,
|
||||
gcs_path: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Start downloading an audio file.
|
||||
|
||||
Args:
|
||||
search_id: Search ID from previous search
|
||||
result_index: Index of result to download
|
||||
output_filename: Optional custom filename (without extension)
|
||||
gcs_path: GCS path for upload (e.g., "uploads/job123/audio/")
|
||||
|
||||
Returns:
|
||||
Download ID for tracking progress
|
||||
|
||||
Raises:
|
||||
FlacfetchServiceError: On download start failure
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
payload = {
|
||||
"search_id": search_id,
|
||||
"result_index": result_index,
|
||||
}
|
||||
if output_filename:
|
||||
payload["output_filename"] = output_filename
|
||||
if gcs_path:
|
||||
payload["upload_to_gcs"] = True
|
||||
payload["gcs_path"] = gcs_path
|
||||
|
||||
resp = await client.post(
|
||||
f"{self.base_url}/download",
|
||||
headers=self._headers(),
|
||||
json=payload,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
|
||||
data = resp.json()
|
||||
return data["download_id"]
|
||||
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"Download request failed: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise FlacfetchServiceError(f"Download start failed: {e.response.status_code} - {e.response.text}")
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Get the current status of a download.
|
||||
|
||||
Args:
|
||||
download_id: Download ID from start_download
|
||||
|
||||
Returns:
|
||||
Status dict with progress, speed, path info
|
||||
|
||||
Raises:
|
||||
FlacfetchServiceError: On status check failure
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{self.base_url}/download/{download_id}/status",
|
||||
headers=self._headers(),
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"Status request failed: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
if e.response.status_code == 404:
|
||||
raise FlacfetchServiceError(f"Download not found: {download_id}")
|
||||
raise FlacfetchServiceError(f"Status check failed: {e.response.status_code}")
|
||||
|
||||
async def wait_for_download(
|
||||
self,
|
||||
download_id: str,
|
||||
timeout: int = 600,
|
||||
poll_interval: float = 2.0,
|
||||
progress_callback: Optional[callable] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Wait for a download to complete.
|
||||
|
||||
Args:
|
||||
download_id: Download ID to wait for
|
||||
timeout: Maximum wait time in seconds
|
||||
poll_interval: Time between status checks
|
||||
progress_callback: Optional callback(status_dict) for progress updates
|
||||
|
||||
Returns:
|
||||
Final status dict with output_path or gcs_path
|
||||
|
||||
Raises:
|
||||
FlacfetchServiceError: On download failure or timeout
|
||||
"""
|
||||
elapsed = 0.0
|
||||
|
||||
while elapsed < timeout:
|
||||
status = await self.get_download_status(download_id)
|
||||
|
||||
if progress_callback:
|
||||
try:
|
||||
progress_callback(status)
|
||||
except Exception as e:
|
||||
logger.warning(f"Progress callback error: {e}")
|
||||
|
||||
download_status = status.get("status")
|
||||
|
||||
if download_status in ["complete", "seeding"]:
|
||||
logger.info(f"Download {download_id} complete: {status.get('gcs_path') or status.get('output_path')}")
|
||||
return status
|
||||
elif download_status == "failed":
|
||||
error = status.get("error", "Unknown error")
|
||||
raise FlacfetchServiceError(f"Download failed: {error}")
|
||||
elif download_status == "cancelled":
|
||||
raise FlacfetchServiceError("Download was cancelled")
|
||||
|
||||
# Log progress
|
||||
progress = status.get("progress", 0)
|
||||
speed = status.get("download_speed_kbps", 0)
|
||||
peers = status.get("peers", 0)
|
||||
logger.debug(
|
||||
f"Download {download_id}: {progress:.1f}% "
|
||||
f"({speed:.1f} KB/s, {peers} peers)"
|
||||
)
|
||||
|
||||
await asyncio.sleep(poll_interval)
|
||||
elapsed += poll_interval
|
||||
|
||||
raise FlacfetchServiceError(f"Download timed out after {timeout}s")
|
||||
|
||||
async def list_torrents(self) -> Dict[str, Any]:
|
||||
"""
|
||||
List all torrents in Transmission.
|
||||
|
||||
Returns:
|
||||
Torrent list response with torrent info and totals
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{self.base_url}/torrents",
|
||||
headers=self._headers(),
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"List torrents failed: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise FlacfetchServiceError(f"List torrents failed: {e.response.status_code}")
|
||||
|
||||
async def delete_torrent(
|
||||
self,
|
||||
torrent_id: int,
|
||||
delete_data: bool = True,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Delete a torrent from Transmission.
|
||||
|
||||
Args:
|
||||
torrent_id: Transmission torrent ID
|
||||
delete_data: Also delete downloaded files
|
||||
|
||||
Returns:
|
||||
Delete response
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.delete(
|
||||
f"{self.base_url}/torrents/{torrent_id}",
|
||||
headers=self._headers(),
|
||||
params={"delete_data": str(delete_data).lower()},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"Delete torrent failed: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise FlacfetchServiceError(f"Delete torrent failed: {e.response.status_code}")
|
||||
|
||||
async def cleanup_torrents(
|
||||
self,
|
||||
strategy: str = "oldest",
|
||||
target_free_gb: float = 10.0,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Trigger disk cleanup.
|
||||
|
||||
Args:
|
||||
strategy: Cleanup strategy (oldest, largest, lowest_ratio)
|
||||
target_free_gb: Target free space to achieve
|
||||
|
||||
Returns:
|
||||
Cleanup response with removed count and freed space
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(
|
||||
f"{self.base_url}/torrents/cleanup",
|
||||
headers=self._headers(),
|
||||
json={
|
||||
"strategy": strategy,
|
||||
"target_free_gb": target_free_gb,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
except httpx.RequestError as e:
|
||||
raise FlacfetchServiceError(f"Cleanup failed: {e}")
|
||||
except httpx.HTTPStatusError as e:
|
||||
raise FlacfetchServiceError(f"Cleanup failed: {e.response.status_code}")
|
||||
|
||||
|
||||
# Singleton client instance
|
||||
_client: Optional[FlacfetchClient] = None
|
||||
|
||||
|
||||
def get_flacfetch_client() -> Optional[FlacfetchClient]:
|
||||
"""
|
||||
Get the flacfetch client if configured.
|
||||
|
||||
Returns:
|
||||
FlacfetchClient instance if FLACFETCH_API_URL and FLACFETCH_API_KEY are set,
|
||||
None otherwise (indicating local-only mode).
|
||||
"""
|
||||
global _client
|
||||
|
||||
if _client is None:
|
||||
from backend.config import get_settings
|
||||
settings = get_settings()
|
||||
|
||||
if settings.flacfetch_api_url and settings.flacfetch_api_key:
|
||||
_client = FlacfetchClient(
|
||||
base_url=settings.flacfetch_api_url,
|
||||
api_key=settings.flacfetch_api_key,
|
||||
)
|
||||
logger.info("FlacfetchClient initialized for remote service")
|
||||
else:
|
||||
logger.debug("FlacfetchClient not configured (missing URL or API key)")
|
||||
|
||||
return _client
|
||||
|
||||
|
||||
def reset_flacfetch_client():
|
||||
"""Reset the singleton client (for testing)."""
|
||||
global _client
|
||||
_client = None
|
||||
|
||||
@@ -99,7 +99,11 @@ class TestJobStatusAudioSearchStates:
|
||||
|
||||
|
||||
class TestAudioSearchResult:
|
||||
"""Test AudioSearchResult dataclass."""
|
||||
"""Test AudioSearchResult dataclass.
|
||||
|
||||
AudioSearchResult is now imported from karaoke_gen.audio_fetcher.
|
||||
These tests verify the backend can use it correctly.
|
||||
"""
|
||||
|
||||
def test_create_audio_search_result(self):
|
||||
"""Test creating an AudioSearchResult."""
|
||||
@@ -140,6 +144,8 @@ class TestAudioSearchResult:
|
||||
assert data['provider'] == "YouTube"
|
||||
assert data['url'] == "https://youtube.com/watch?v=abc123"
|
||||
assert data['index'] == 0
|
||||
# raw_result should NOT be in serialized dict
|
||||
assert 'raw_result' not in data
|
||||
|
||||
def test_from_dict(self):
|
||||
"""Test creating from dict."""
|
||||
@@ -160,6 +166,7 @@ class TestAudioSearchResult:
|
||||
assert result.artist == "ABBA"
|
||||
assert result.provider == "YouTube"
|
||||
assert result.index == 0
|
||||
assert result.raw_result is None # Not set from dict
|
||||
|
||||
|
||||
class TestAudioSearchServiceInit:
|
||||
@@ -171,10 +178,9 @@ class TestAudioSearchServiceInit:
|
||||
# Pass explicit None to override environment
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
|
||||
# When explicitly passed None, it should be None (not from env)
|
||||
assert service._redacted_api_key is None
|
||||
assert service._ops_api_key is None
|
||||
assert service._manager is None
|
||||
# Service should have a FlacFetcher internally
|
||||
assert service._fetcher is not None
|
||||
assert service._cached_results == []
|
||||
|
||||
def test_init_with_keys(self):
|
||||
"""Test initialization with API keys."""
|
||||
@@ -183,41 +189,40 @@ class TestAudioSearchServiceInit:
|
||||
ops_api_key="test_ops_key",
|
||||
)
|
||||
|
||||
assert service._redacted_api_key == "test_redacted_key"
|
||||
assert service._ops_api_key == "test_ops_key"
|
||||
# Keys are passed to FlacFetcher
|
||||
assert service._fetcher._redacted_api_key == "test_redacted_key"
|
||||
assert service._fetcher._ops_api_key == "test_ops_key"
|
||||
|
||||
def test_init_reads_from_environment(self):
|
||||
"""Test initialization reads keys from environment variables."""
|
||||
import os
|
||||
|
||||
# Just test that the service can be initialized
|
||||
service = AudioSearchService()
|
||||
|
||||
# The service should have API keys from env or None
|
||||
assert service._manager is None # Not yet initialized
|
||||
# The service should have a FlacFetcher
|
||||
assert service._fetcher is not None
|
||||
|
||||
|
||||
class TestAudioSearchServiceSearch:
|
||||
"""Test AudioSearchService.search() method."""
|
||||
|
||||
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
|
||||
def test_search_returns_results(self, mock_get_manager):
|
||||
def test_search_returns_results(self):
|
||||
"""Test search returns AudioSearchResult list."""
|
||||
# Mock the flacfetch result
|
||||
mock_result = Mock()
|
||||
mock_result.title = "Waterloo"
|
||||
mock_result.artist = "ABBA"
|
||||
mock_result.provider = "YouTube"
|
||||
mock_result.url = "https://youtube.com/watch?v=abc123"
|
||||
mock_result.duration = 180
|
||||
mock_result.quality = "FLAC"
|
||||
mock_result.id = "abc123"
|
||||
# Create mock result
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="YouTube",
|
||||
url="https://youtube.com/watch?v=abc123",
|
||||
duration=180,
|
||||
quality="FLAC",
|
||||
source_id="abc123",
|
||||
index=0,
|
||||
)
|
||||
|
||||
mock_manager = Mock()
|
||||
mock_manager.search.return_value = [mock_result]
|
||||
mock_get_manager.return_value = mock_manager
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.search.return_value = [mock_result]
|
||||
|
||||
service = AudioSearchService()
|
||||
results = service.search("ABBA", "Waterloo")
|
||||
|
||||
assert len(results) == 1
|
||||
@@ -226,40 +231,36 @@ class TestAudioSearchServiceSearch:
|
||||
assert results[0].provider == "YouTube"
|
||||
assert results[0].index == 0
|
||||
|
||||
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
|
||||
def test_search_no_results_raises_error(self, mock_get_manager):
|
||||
def test_search_no_results_raises_error(self):
|
||||
"""Test search raises NoResultsError when no results."""
|
||||
mock_manager = Mock()
|
||||
mock_manager.search.return_value = []
|
||||
mock_get_manager.return_value = mock_manager
|
||||
|
||||
service = AudioSearchService()
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.search.side_effect = NoResultsError("No results found")
|
||||
|
||||
with pytest.raises(NoResultsError) as exc_info:
|
||||
service.search("Unknown Artist", "Unknown Song")
|
||||
|
||||
assert "No results found" in str(exc_info.value)
|
||||
|
||||
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
|
||||
def test_search_multiple_results(self, mock_get_manager):
|
||||
def test_search_multiple_results(self):
|
||||
"""Test search returns multiple results with correct indices."""
|
||||
mock_results = []
|
||||
for i in range(3):
|
||||
mock_result = Mock()
|
||||
mock_result.title = f"Song {i}"
|
||||
mock_result.artist = "Artist"
|
||||
mock_result.provider = ["YouTube", "Redacted", "OPS"][i]
|
||||
mock_result.url = f"https://example.com/{i}"
|
||||
mock_result.duration = 180 + i * 10
|
||||
mock_result.quality = ["320kbps", "FLAC", "FLAC"][i]
|
||||
mock_result.id = str(i)
|
||||
mock_results.append(mock_result)
|
||||
mock_results.append(AudioSearchResult(
|
||||
title=f"Song {i}",
|
||||
artist="Artist",
|
||||
provider=["YouTube", "Redacted", "OPS"][i],
|
||||
url=f"https://example.com/{i}",
|
||||
duration=180 + i * 10,
|
||||
quality=["320kbps", "FLAC", "FLAC"][i],
|
||||
source_id=str(i),
|
||||
index=i,
|
||||
))
|
||||
|
||||
mock_manager = Mock()
|
||||
mock_manager.search.return_value = mock_results
|
||||
mock_get_manager.return_value = mock_manager
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.search.return_value = mock_results
|
||||
|
||||
service = AudioSearchService()
|
||||
results = service.search("Artist", "Song")
|
||||
|
||||
assert len(results) == 3
|
||||
@@ -271,34 +272,39 @@ class TestAudioSearchServiceSearch:
|
||||
class TestAudioSearchServiceSelectBest:
|
||||
"""Test AudioSearchService.select_best() method."""
|
||||
|
||||
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
|
||||
def test_select_best_uses_manager(self, mock_get_manager):
|
||||
"""Test select_best uses flacfetch's select_best."""
|
||||
# First need to do a search to populate cache
|
||||
mock_result = Mock()
|
||||
mock_result.title = "Waterloo"
|
||||
mock_result.artist = "ABBA"
|
||||
mock_result.provider = "YouTube"
|
||||
mock_result.url = "https://youtube.com/watch?v=abc123"
|
||||
mock_result.duration = 180
|
||||
mock_result.quality = "FLAC"
|
||||
mock_result.id = "abc123"
|
||||
def test_select_best_delegates_to_fetcher(self):
|
||||
"""Test select_best uses FlacFetcher's select_best."""
|
||||
mock_results = [
|
||||
AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="YouTube",
|
||||
url="https://youtube.com/watch?v=abc123",
|
||||
index=0,
|
||||
),
|
||||
AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="Redacted",
|
||||
url="https://example.com/456",
|
||||
index=1,
|
||||
),
|
||||
]
|
||||
|
||||
mock_manager = Mock()
|
||||
mock_manager.search.return_value = [mock_result]
|
||||
mock_manager.select_best.return_value = mock_result
|
||||
mock_get_manager.return_value = mock_manager
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.select_best.return_value = 1
|
||||
|
||||
service = AudioSearchService()
|
||||
results = service.search("ABBA", "Waterloo")
|
||||
best_index = service.select_best(mock_results)
|
||||
|
||||
best_index = service.select_best(results)
|
||||
|
||||
assert best_index == 0
|
||||
assert best_index == 1
|
||||
service._fetcher.select_best.assert_called_once_with(mock_results)
|
||||
|
||||
def test_select_best_empty_list_returns_zero(self):
|
||||
"""Test select_best returns 0 for empty list."""
|
||||
service = AudioSearchService()
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.select_best.return_value = 0
|
||||
|
||||
best_index = service.select_best([])
|
||||
|
||||
@@ -308,24 +314,18 @@ class TestAudioSearchServiceSelectBest:
|
||||
class TestAudioSearchServiceDownload:
|
||||
"""Test AudioSearchService.download() method."""
|
||||
|
||||
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
|
||||
def test_download_without_search_raises_error(self, mock_get_manager):
|
||||
def test_download_without_search_raises_error(self):
|
||||
"""Test download without prior search raises error."""
|
||||
mock_manager = Mock()
|
||||
mock_get_manager.return_value = mock_manager
|
||||
|
||||
service = AudioSearchService()
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
|
||||
with pytest.raises(DownloadError) as exc_info:
|
||||
service.download(0, "/tmp")
|
||||
|
||||
assert "No cached result" in str(exc_info.value)
|
||||
|
||||
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
|
||||
def test_download_after_search(self, mock_get_manager):
|
||||
def test_download_after_search(self):
|
||||
"""Test download after search works."""
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
# Create a temp file to simulate downloaded file
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
@@ -333,23 +333,32 @@ class TestAudioSearchServiceDownload:
|
||||
with open(temp_file, 'w') as f:
|
||||
f.write("test")
|
||||
|
||||
mock_result = Mock()
|
||||
mock_result.title = "Waterloo"
|
||||
mock_result.artist = "ABBA"
|
||||
mock_result.provider = "YouTube"
|
||||
mock_result.url = "https://youtube.com/watch?v=abc123"
|
||||
mock_result.duration = 180
|
||||
mock_result.quality = "FLAC"
|
||||
mock_result.id = "abc123"
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="YouTube",
|
||||
url="https://youtube.com/watch?v=abc123",
|
||||
duration=180,
|
||||
quality="FLAC",
|
||||
source_id="abc123",
|
||||
index=0,
|
||||
)
|
||||
|
||||
mock_manager = Mock()
|
||||
mock_manager.search.return_value = [mock_result]
|
||||
mock_manager.download.return_value = temp_file
|
||||
mock_get_manager.return_value = mock_manager
|
||||
mock_fetch_result = AudioDownloadResult(
|
||||
filepath=temp_file,
|
||||
artist="ABBA",
|
||||
title="Waterloo",
|
||||
provider="YouTube",
|
||||
duration=180,
|
||||
quality="FLAC",
|
||||
)
|
||||
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.search.return_value = [mock_result]
|
||||
service._fetcher.download.return_value = mock_fetch_result
|
||||
|
||||
service = AudioSearchService()
|
||||
service.search("ABBA", "Waterloo")
|
||||
|
||||
result = service.download(0, temp_dir)
|
||||
|
||||
assert result.filepath == temp_file
|
||||
@@ -360,6 +369,27 @@ class TestAudioSearchServiceDownload:
|
||||
# Cleanup
|
||||
os.remove(temp_file)
|
||||
os.rmdir(temp_dir)
|
||||
|
||||
def test_download_invalid_index_raises_error(self):
|
||||
"""Test download with invalid index raises error."""
|
||||
mock_result = AudioSearchResult(
|
||||
title="Waterloo",
|
||||
artist="ABBA",
|
||||
provider="YouTube",
|
||||
url="https://youtube.com/watch?v=abc123",
|
||||
index=0,
|
||||
)
|
||||
|
||||
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
|
||||
service._fetcher = Mock()
|
||||
service._fetcher.search.return_value = [mock_result]
|
||||
|
||||
service.search("ABBA", "Waterloo")
|
||||
|
||||
with pytest.raises(DownloadError) as exc_info:
|
||||
service.download(99, "/tmp")
|
||||
|
||||
assert "No cached result for index 99" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestGetAudioSearchService:
|
||||
@@ -384,3 +414,86 @@ class TestGetAudioSearchService:
|
||||
assert service1 is service2
|
||||
|
||||
|
||||
class TestFlacfetchIntegration:
|
||||
"""
|
||||
Integration tests that verify flacfetch imports work correctly.
|
||||
|
||||
These tests ensure the backend code is compatible with the installed
|
||||
flacfetch version and would catch issues like renamed classes.
|
||||
|
||||
Note: Some tests are marked with pytest.importorskip for flacfetch
|
||||
since it may not be installed in all test environments (e.g., CI).
|
||||
"""
|
||||
|
||||
def test_flacfetch_imports_work(self):
|
||||
"""Test that all flacfetch imports in audio_search_service work.
|
||||
|
||||
This test would have caught the YouTubeProvider -> YoutubeProvider rename.
|
||||
"""
|
||||
flacfetch = pytest.importorskip("flacfetch")
|
||||
|
||||
# These imports match what karaoke_gen.audio_fetcher does
|
||||
from flacfetch.core.manager import FetchManager
|
||||
from flacfetch.providers.youtube import YoutubeProvider
|
||||
from flacfetch.core.models import TrackQuery
|
||||
|
||||
# Verify the classes exist and are callable
|
||||
assert FetchManager is not None
|
||||
assert YoutubeProvider is not None
|
||||
assert TrackQuery is not None
|
||||
|
||||
def test_karaoke_gen_audio_fetcher_imports_work(self):
|
||||
"""Test that karaoke_gen.audio_fetcher imports work.
|
||||
|
||||
Since backend now imports from karaoke_gen, this is the critical test.
|
||||
This should work even without flacfetch installed (lazy import).
|
||||
"""
|
||||
from karaoke_gen.audio_fetcher import (
|
||||
FlacFetcher,
|
||||
AudioSearchResult,
|
||||
AudioFetchResult,
|
||||
AudioFetcherError,
|
||||
NoResultsError,
|
||||
DownloadError,
|
||||
)
|
||||
|
||||
assert FlacFetcher is not None
|
||||
assert AudioSearchResult is not None
|
||||
assert AudioFetchResult is not None
|
||||
assert AudioFetcherError is not None
|
||||
assert NoResultsError is not None
|
||||
assert DownloadError is not None
|
||||
|
||||
def test_audio_search_service_can_initialize_fetcher(self):
|
||||
"""Test AudioSearchService can initialize its FlacFetcher.
|
||||
|
||||
This verifies the actual initialization code path works.
|
||||
"""
|
||||
pytest.importorskip("flacfetch")
|
||||
|
||||
service = AudioSearchService(
|
||||
redacted_api_key=None,
|
||||
ops_api_key=None,
|
||||
)
|
||||
|
||||
# Verify FlacFetcher was initialized
|
||||
assert service._fetcher is not None
|
||||
|
||||
# FlacFetcher can initialize its manager (tests actual flacfetch)
|
||||
manager = service._fetcher._get_manager()
|
||||
assert manager is not None
|
||||
|
||||
def test_shared_classes_are_same_as_karaoke_gen(self):
|
||||
"""Verify backend uses the same classes as karaoke_gen."""
|
||||
from karaoke_gen.audio_fetcher import (
|
||||
AudioSearchResult as KGAudioSearchResult,
|
||||
AudioFetchResult as KGAudioFetchResult,
|
||||
NoResultsError as KGNoResultsError,
|
||||
DownloadError as KGDownloadError,
|
||||
)
|
||||
|
||||
# These should be the exact same classes, not copies
|
||||
assert AudioSearchResult is KGAudioSearchResult
|
||||
assert AudioDownloadResult is KGAudioFetchResult
|
||||
assert NoResultsError is KGNoResultsError
|
||||
assert DownloadError is KGDownloadError
|
||||
@@ -447,11 +447,24 @@ async def upload_lyrics_results(
|
||||
corrections_data = json.load(f)
|
||||
|
||||
# Store metadata in state_data
|
||||
job_manager.update_state_data(job_id, 'lyrics_metadata', {
|
||||
# Include countdown padding info for instrumental synchronization
|
||||
lyrics_metadata = {
|
||||
'segment_count': len(corrections_data.get('corrected_segments', [])),
|
||||
'has_corrections': True,
|
||||
'ready_for_review': True
|
||||
})
|
||||
'ready_for_review': True,
|
||||
}
|
||||
|
||||
# Add countdown padding info from transcription result
|
||||
# This is needed by video_worker to pad instrumentals for sync
|
||||
if transcription_result.get("countdown_padding_added"):
|
||||
lyrics_metadata['has_countdown_padding'] = True
|
||||
lyrics_metadata['countdown_padding_seconds'] = transcription_result.get("countdown_padding_seconds", 3.0)
|
||||
logger.info(f"Job {job_id}: Countdown padding detected: {lyrics_metadata['countdown_padding_seconds']}s")
|
||||
else:
|
||||
lyrics_metadata['has_countdown_padding'] = False
|
||||
lyrics_metadata['countdown_padding_seconds'] = 0.0
|
||||
|
||||
job_manager.update_state_data(job_id, 'lyrics_metadata', lyrics_metadata)
|
||||
except Exception as e:
|
||||
logger.warning(f"Job {job_id}: Could not parse corrections JSON: {e}")
|
||||
else:
|
||||
|
||||
@@ -162,6 +162,14 @@ async def generate_video(job_id: str) -> bool:
|
||||
message="Encoding videos (15-20 min)"
|
||||
)
|
||||
|
||||
# Get countdown padding info from lyrics metadata
|
||||
# This ensures instrumental is padded to match vocals if countdown was added
|
||||
lyrics_metadata = job.state_data.get('lyrics_metadata', {})
|
||||
countdown_padding_seconds = None
|
||||
if lyrics_metadata.get('has_countdown_padding'):
|
||||
countdown_padding_seconds = lyrics_metadata.get('countdown_padding_seconds', 3.0)
|
||||
job_log.info(f"Countdown padding detected: {countdown_padding_seconds}s - instrumental will be padded if needed")
|
||||
|
||||
# Log finalization parameters
|
||||
job_log.info("Creating KaraokeFinalise with parameters:")
|
||||
job_log.info(f" enable_cdg: {getattr(job, 'enable_cdg', False)}")
|
||||
@@ -169,6 +177,7 @@ async def generate_video(job_id: str) -> bool:
|
||||
job_log.info(f" brand_prefix: {getattr(job, 'brand_prefix', None)}")
|
||||
job_log.info(f" discord_webhook: {'configured' if getattr(job, 'discord_webhook_url', None) else 'not configured'}")
|
||||
job_log.info(f" instrumental source: {instrumental_source}")
|
||||
job_log.info(f" countdown_padding_seconds: {countdown_padding_seconds}")
|
||||
if existing_instrumental_path:
|
||||
job_log.info(f" using user-provided instrumental (selection was: {instrumental_selection})")
|
||||
|
||||
@@ -209,6 +218,8 @@ async def generate_video(job_id: str) -> bool:
|
||||
non_interactive=True,
|
||||
server_side_mode=True,
|
||||
selected_instrumental_file=instrumental_file,
|
||||
# Audio synchronization - ensure instrumental matches vocal padding
|
||||
countdown_padding_seconds=countdown_padding_seconds,
|
||||
)
|
||||
|
||||
# Call process() - this does ALL the work:
|
||||
|
||||
@@ -1,22 +1,23 @@
|
||||
# Cloud Build configuration for base image
|
||||
#
|
||||
# This builds the base image with all Python dependencies pre-installed.
|
||||
# Run this manually when poetry.lock changes:
|
||||
# Run this manually when poetry.lock OR Dockerfile.base changes:
|
||||
# gcloud builds submit --config=cloudbuild-base.yaml --project=nomadkaraoke
|
||||
#
|
||||
# The base image is used by the main Dockerfile for fast app builds.
|
||||
# The poetry.lock hash is stored as a label for change detection.
|
||||
# A combined hash of poetry.lock + Dockerfile.base is stored as a label for change detection.
|
||||
|
||||
steps:
|
||||
# Calculate poetry.lock hash for change detection
|
||||
# Calculate combined hash for change detection (poetry.lock + Dockerfile.base)
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
entrypoint: 'bash'
|
||||
args:
|
||||
- '-c'
|
||||
- |
|
||||
POETRY_LOCK_HASH=$$(sha256sum poetry.lock | cut -d' ' -f1)
|
||||
echo "Poetry.lock hash: $$POETRY_LOCK_HASH"
|
||||
echo "$$POETRY_LOCK_HASH" > /workspace/POETRY_LOCK_HASH
|
||||
# Hash both files to detect changes in either
|
||||
CONTENT_HASH=$$(cat poetry.lock backend/Dockerfile.base | sha256sum | cut -d' ' -f1)
|
||||
echo "Combined content hash (poetry.lock + Dockerfile.base): $$CONTENT_HASH"
|
||||
echo "$$CONTENT_HASH" > /workspace/CONTENT_HASH
|
||||
|
||||
# Build base image with all dependencies
|
||||
- name: 'gcr.io/cloud-builders/docker'
|
||||
@@ -24,11 +25,11 @@ steps:
|
||||
args:
|
||||
- '-c'
|
||||
- |
|
||||
POETRY_LOCK_HASH=$$(cat /workspace/POETRY_LOCK_HASH)
|
||||
CONTENT_HASH=$$(cat /workspace/CONTENT_HASH)
|
||||
docker build \
|
||||
--build-arg BUILD_DATE=$BUILD_ID \
|
||||
--build-arg BUILD_ID=$BUILD_ID \
|
||||
--build-arg POETRY_LOCK_HASH=$$POETRY_LOCK_HASH \
|
||||
--build-arg BASE_CONTENT_HASH=$$CONTENT_HASH \
|
||||
-t us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:latest \
|
||||
-t us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:$BUILD_ID \
|
||||
-f backend/Dockerfile.base \
|
||||
|
||||
+1
-1
@@ -76,7 +76,7 @@ steps:
|
||||
--max-instances 50 \
|
||||
--min-instances 0 \
|
||||
--set-env-vars "GOOGLE_CLOUD_PROJECT=$PROJECT_ID,GCS_BUCKET_NAME=karaoke-gen-storage-$PROJECT_ID,FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,AUDIO_SEPARATOR_API_URL=https://nomadkaraoke--audio-separator-api.modal.run,KARAOKE_GEN_VERSION=$$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1" \
|
||||
--set-secrets "AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest"
|
||||
--set-secrets "AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,REDACTED_API_KEY=redacted-api-key:latest,OPS_API_KEY=ops-api-key:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest"
|
||||
|
||||
images:
|
||||
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:latest'
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
# Codecov Configuration
|
||||
# Docs: https://docs.codecov.com/docs/codecovyml-reference
|
||||
|
||||
coverage:
|
||||
# Coverage value precision (2 decimal places)
|
||||
precision: 2
|
||||
round: down
|
||||
# Green range starts at 70% (our target)
|
||||
range: "70...100"
|
||||
|
||||
status:
|
||||
# Project-level coverage status check
|
||||
project:
|
||||
default:
|
||||
target: 70% # Pass if coverage >= 70%
|
||||
threshold: 2% # Allow 2% drop without failing
|
||||
if_ci_failed: error
|
||||
|
||||
# Patch coverage (new/changed lines only)
|
||||
patch:
|
||||
default:
|
||||
target: 70% # New code should also meet 70%
|
||||
threshold: 5% # More lenient for patches
|
||||
|
||||
# Disable PR comments entirely since we're satisfied with 70%+ coverage
|
||||
# Status checks will still appear on the PR
|
||||
comment: false
|
||||
|
||||
# Alternative: Uncomment below to only comment when coverage changes
|
||||
# comment:
|
||||
# layout: "diff, flags, files"
|
||||
# behavior: default
|
||||
# require_changes: true # Only comment if coverage actually changes
|
||||
# require_base: true # Require base report to exist
|
||||
|
||||
@@ -681,6 +681,8 @@ The karaoke-gen system supports multiple interfaces to the same core functionali
|
||||
| `dropbox-oauth-credentials` | ✅ | Dropbox API OAuth tokens |
|
||||
| `gdrive-service-account` | ✅ | Google Drive service account |
|
||||
| `discord-webhook-url` | ✅ | Default Discord webhook |
|
||||
| `redacted-api-key` | ⏳ | Redacted tracker API key (flacfetch audio search) |
|
||||
| `ops-api-key` | ⏳ | OPS/Orpheus tracker API key (flacfetch audio search) |
|
||||
|
||||
### Default Distribution Settings
|
||||
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
# Refactor: Unify Audio Fetcher Code
|
||||
|
||||
**Created:** 2025-12-20
|
||||
**Completed:** 2025-12-20
|
||||
**Priority:** MEDIUM (prevents future bugs like YouTubeProvider -> YoutubeProvider)
|
||||
**Status:** ✅ Completed
|
||||
|
||||
## Problem
|
||||
|
||||
The codebase has **two separate implementations** of flacfetch integration:
|
||||
1. `karaoke_gen/audio_fetcher.py` - Used by local CLI
|
||||
2. `backend/services/audio_search_service.py` - Used by cloud backend
|
||||
|
||||
This violates DRY and caused a bug where the backend's import wasn't updated when flacfetch renamed `YouTubeProvider` to `YoutubeProvider`.
|
||||
|
||||
## Current State
|
||||
|
||||
### Duplicated Code
|
||||
|
||||
| Component | Local (`audio_fetcher.py`) | Backend (`audio_search_service.py`) |
|
||||
|-----------|---------------------------|-------------------------------------|
|
||||
| `_get_manager()` | ✅ 40 lines | ❌ 35 lines (copy) |
|
||||
| Provider setup | ✅ Redacted, OPS, YouTube | ❌ Same, diverged |
|
||||
| `search()` | ✅ Full implementation | ❌ Similar, diverged |
|
||||
| `download()` | ✅ With downloaders | ⚠️ Different, simpler |
|
||||
| `AudioSearchResult` | ✅ Dataclass | ❌ Duplicate, different fields |
|
||||
| `AudioDownloadResult` | ✅ (`AudioFetchResult`) | ❌ Duplicate |
|
||||
| Exceptions | ✅ 4 types | ❌ 3 types, different names |
|
||||
|
||||
### Drift Examples
|
||||
- Backend missing `raw_result` field on `AudioSearchResult`
|
||||
- Backend has `index` field, local doesn't
|
||||
- Backend exceptions inherit from `AudioSearchError`, local from `AudioFetcherError`
|
||||
- Backend missing `UserCancelledError`
|
||||
- Backend missing downloader registration (TorrentDownloader, YoutubeDownloader)
|
||||
|
||||
## Proposed Solution
|
||||
|
||||
### Option 1: Backend imports from karaoke_gen (Recommended)
|
||||
|
||||
The backend already depends on `karaoke_gen` (installed via `pip install -e .` in Dockerfile).
|
||||
|
||||
```python
|
||||
# backend/services/audio_search_service.py - REFACTORED
|
||||
|
||||
from karaoke_gen.audio_fetcher import (
|
||||
FlacFetcher,
|
||||
AudioSearchResult,
|
||||
AudioFetchResult,
|
||||
AudioFetcherError,
|
||||
NoResultsError,
|
||||
DownloadError,
|
||||
)
|
||||
|
||||
class AudioSearchService:
|
||||
"""
|
||||
Thin wrapper around karaoke_gen.audio_fetcher.FlacFetcher.
|
||||
|
||||
Adds backend-specific functionality:
|
||||
- Firestore serialization (to_dict/from_dict)
|
||||
- GCS integration for downloads
|
||||
- Job state management
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
redacted_api_key: Optional[str] = None,
|
||||
ops_api_key: Optional[str] = None,
|
||||
):
|
||||
self._fetcher = FlacFetcher(
|
||||
redacted_api_key=redacted_api_key,
|
||||
ops_api_key=ops_api_key,
|
||||
)
|
||||
|
||||
def search(self, artist: str, title: str) -> List[AudioSearchResult]:
|
||||
"""Delegate to FlacFetcher.search()."""
|
||||
return self._fetcher.search(artist, title)
|
||||
|
||||
def select_best(self, results: List[AudioSearchResult]) -> int:
|
||||
"""Delegate to FlacFetcher.select_best()."""
|
||||
return self._fetcher.select_best(results)
|
||||
|
||||
def download(
|
||||
self,
|
||||
result: AudioSearchResult,
|
||||
output_dir: str,
|
||||
output_filename: Optional[str] = None,
|
||||
) -> AudioFetchResult:
|
||||
"""Delegate to FlacFetcher.download()."""
|
||||
return self._fetcher.download(result, output_dir, output_filename)
|
||||
```
|
||||
|
||||
### Changes Required to karaoke_gen.audio_fetcher
|
||||
|
||||
1. **Add `to_dict()` method to `AudioSearchResult`** - For Firestore serialization
|
||||
2. **Add `from_dict()` classmethod** - For deserialization
|
||||
3. **Ensure `select_best()` is a standalone method** - Currently buried in interactive flow
|
||||
4. **Export all public classes from `__init__.py`**
|
||||
|
||||
### Changes Required to backend
|
||||
|
||||
1. **Remove duplicate dataclasses** - Use imports from karaoke_gen
|
||||
2. **Remove duplicate `_get_manager()`** - Delegate to FlacFetcher
|
||||
3. **Remove duplicate exception classes** - Use karaoke_gen exceptions
|
||||
4. **Update tests** - Point to real classes, not mocks
|
||||
|
||||
## Benefits
|
||||
|
||||
1. **Single source of truth** - Bug fixes apply everywhere
|
||||
2. **No more drift** - Dataclasses, exceptions stay in sync
|
||||
3. **Better test coverage** - karaoke_gen has integration tests for flacfetch imports
|
||||
4. **Less code to maintain** - Backend becomes thin wrapper
|
||||
|
||||
## Migration Steps
|
||||
|
||||
1. [x] Add `to_dict()` / `from_dict()` to `AudioSearchResult` in karaoke_gen
|
||||
2. [x] Extract `select_best()` as standalone method in FlacFetcher
|
||||
3. [x] Update karaoke_gen `__init__.py` exports (with lazy imports)
|
||||
4. [x] Refactor backend to import from karaoke_gen
|
||||
5. [x] Update backend tests
|
||||
6. [x] Delete duplicate code from backend (~150 lines removed)
|
||||
7. [x] Add integration test that catches import divergence
|
||||
|
||||
## Related Files
|
||||
|
||||
- `karaoke_gen/audio_fetcher.py` - Source of truth (keep)
|
||||
- `backend/services/audio_search_service.py` - Refactor to thin wrapper
|
||||
- `backend/api/routes/audio_search.py` - Update imports
|
||||
- `backend/tests/test_audio_search.py` - Update tests
|
||||
|
||||
## Estimated Effort
|
||||
|
||||
- **Time:** 2-4 hours
|
||||
- **Risk:** Low (well-defined changes, good test coverage)
|
||||
- **Dependencies:** None
|
||||
|
||||
## Related Architecture Docs
|
||||
|
||||
- `docs/01-reference/ARCHITECTURE.md` - "LyricsTranscriber Integration" section shows similar pattern
|
||||
- `docs/00-current-plan/SCALABLE-ARCHITECTURE-PLAN.md` - "Shared pipeline architecture" goal
|
||||
|
||||
@@ -0,0 +1,729 @@
|
||||
# Flacfetch HTTP API Service Implementation Plan
|
||||
|
||||
**Created:** 2025-12-20
|
||||
**Updated:** 2025-12-20
|
||||
**Status:** In Progress
|
||||
**Priority:** Medium
|
||||
**Estimated Effort:** 3-4 days
|
||||
|
||||
## Problem Statement
|
||||
|
||||
BitTorrent downloads don't work reliably in Cloud Run due to:
|
||||
- No inbound peer connections (NAT/firewall)
|
||||
- Ephemeral containers with no stable IP
|
||||
- DHT peer discovery blocked
|
||||
- Container timeout limits (max 60 min)
|
||||
|
||||
This prevents downloading lossless audio from private trackers (Redacted, OPS) via the remote backend.
|
||||
|
||||
## Solution Overview
|
||||
|
||||
Add an HTTP API mode to **flacfetch** (our audio fetching library) and deploy it on a dedicated GCP VM with:
|
||||
- Full BitTorrent peer connectivity via Transmission daemon
|
||||
- Static IP for tracker whitelist (seedbox-like)
|
||||
- Indefinite seeding of downloaded torrents
|
||||
- Automatic disk cleanup when space runs low
|
||||
- YouTube downloads also handled (single service for all audio acquisition)
|
||||
|
||||
This approach provides clean separation of concerns:
|
||||
- **flacfetch**: Handles ALL audio acquisition (torrents + YouTube)
|
||||
- **karaoke-gen backend**: Orchestrates karaoke generation pipeline, calls flacfetch API
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────────────────────────────────────────────┐
|
||||
│ FLACFETCH SERVICE ARCHITECTURE │
|
||||
├─────────────────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ Cloud Run Backend Flacfetch VM (e2-small) │
|
||||
│ ┌────────────────────┐ ┌─────────────────────────┐ │
|
||||
│ │ │ │ │ │
|
||||
│ │ karaoke-gen │ 1. Search │ flacfetch HTTP API │ │
|
||||
│ │ AudioSearchService │ ────────────▶ │ POST /search │ │
|
||||
│ │ │ │ │ │
|
||||
│ │ │ 2. Results │ Providers: │ │
|
||||
│ │ │ ◀──────────── │ • Redacted (FLAC) │ │
|
||||
│ │ │ │ • OPS (FLAC) │ │
|
||||
│ │ │ 3. Download │ • YouTube (fallback) │ │
|
||||
│ │ │ ────────────▶ │ │ │
|
||||
│ │ │ │ POST /download │ │
|
||||
│ │ │ 4. Status │ GET /download/{id} │ │
|
||||
│ │ │ ◀──────────── │ │ │
|
||||
│ │ │ (polling) │ Transmission Daemon │ │
|
||||
│ │ │ │ • Full peer connect │ │
|
||||
│ │ │ 5. Complete │ • DHT enabled │ │
|
||||
│ │ │ ◀──────────── │ • Indefinite seeding │ │
|
||||
│ │ │ (file path) │ │ │
|
||||
│ └────────────────────┘ └───────────┬─────────────┘ │
|
||||
│ │ │ │
|
||||
│ │ │ 6. Upload to GCS │
|
||||
│ │ │ │
|
||||
│ ▼ ▼ │
|
||||
│ ┌─────────────────────────────────────────────────────────────────┐ │
|
||||
│ │ Google Cloud Storage │ │
|
||||
│ │ │ │
|
||||
│ │ uploads/{job_id}/audio/{filename}.flac │ │
|
||||
│ │ │ │
|
||||
│ └─────────────────────────────────────────────────────────────────┘ │
|
||||
│ │
|
||||
└─────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Key Design Decisions
|
||||
|
||||
### 1. HTTP API in flacfetch (not separate service)
|
||||
- **Why**: Clean separation of concerns, reusable by future projects, single source of truth
|
||||
- **How**: Add `flacfetch/api/` module with FastAPI, new CLI mode `flacfetch serve`
|
||||
|
||||
### 2. Indefinite Seeding
|
||||
- **Why**: VM is always-on anyway, helps maintain tracker ratio, acts like a seedbox
|
||||
- **Current behavior**: Torrent removed after download (line 317-321 in torrent.py)
|
||||
- **New behavior**: Torrent kept seeding, only removed by cleanup or explicit delete
|
||||
|
||||
### 3. Static IP
|
||||
- **Why**: Required for tracker account IP whitelist, consistent for firewall rules
|
||||
- **Cost**: ~$3/month additional
|
||||
|
||||
### 4. Disk Space Management
|
||||
- **Strategy**: Auto-cleanup oldest torrents when free space < 5GB threshold
|
||||
- **API**: Manual cleanup endpoints + automatic background cleanup
|
||||
|
||||
### 5. Instance Type: e2-small
|
||||
- **Specs**: 0.5 vCPU, 2GB RAM, ~$12/month
|
||||
- **Why**: Sufficient for I/O-bound workload, newer C3/N4 require 2+ vCPUs (overkill)
|
||||
- **Disk**: 30GB SSD (20GB for torrents + headroom)
|
||||
|
||||
### 6. Authentication
|
||||
- **Method**: Static API key in `X-API-Key` header
|
||||
- **Firewall**: Permissive (allows future web UI), auth enforced at app level
|
||||
|
||||
## Implementation Phases
|
||||
|
||||
### Phase 1: Add HTTP API to flacfetch
|
||||
|
||||
**Location:** `./flacfetch/flacfetch/api/`
|
||||
|
||||
**New Directory Structure:**
|
||||
```
|
||||
flacfetch/
|
||||
├── flacfetch/
|
||||
│ ├── api/ # NEW
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── main.py # FastAPI app entry point
|
||||
│ │ ├── auth.py # API key authentication
|
||||
│ │ ├── models.py # Pydantic request/response models
|
||||
│ │ ├── routes/
|
||||
│ │ │ ├── __init__.py
|
||||
│ │ │ ├── search.py # POST /search
|
||||
│ │ │ ├── download.py # POST /download, GET /download/{id}/status
|
||||
│ │ │ ├── torrents.py # GET /torrents, DELETE /torrents/{id}
|
||||
│ │ │ └── health.py # GET /health, GET /disk-usage
|
||||
│ │ └── services/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── download_manager.py # Track active downloads
|
||||
│ │ └── disk_manager.py # Disk cleanup logic
|
||||
│ ├── core/
|
||||
│ ├── downloaders/
|
||||
│ │ └── torrent.py # MODIFIED: optional keep-seeding mode
|
||||
│ ├── interface/
|
||||
│ │ └── cli.py # MODIFIED: add 'serve' subcommand
|
||||
│ └── providers/
|
||||
└── pyproject.toml # MODIFIED: add fastapi, uvicorn deps
|
||||
```
|
||||
|
||||
**API Endpoints:**
|
||||
|
||||
```yaml
|
||||
# All endpoints require X-API-Key header
|
||||
|
||||
# Search for audio
|
||||
POST /search
|
||||
Request:
|
||||
{ "artist": "ABBA", "title": "Waterloo" }
|
||||
Response:
|
||||
{
|
||||
"search_id": "abc123",
|
||||
"results": [
|
||||
{
|
||||
"index": 0,
|
||||
"title": "Waterloo",
|
||||
"artist": "ABBA",
|
||||
"provider": "Redacted",
|
||||
"quality": "FLAC 16bit CD",
|
||||
"seeders": 45,
|
||||
"size_bytes": 31457280,
|
||||
"target_file": "02. Waterloo.flac",
|
||||
...
|
||||
},
|
||||
...
|
||||
]
|
||||
}
|
||||
|
||||
# Start download
|
||||
POST /download
|
||||
Request:
|
||||
{
|
||||
"search_id": "abc123", # From previous search
|
||||
"result_index": 0, # Which result to download
|
||||
"output_filename": "ABBA - Waterloo", # Optional
|
||||
"upload_to_gcs": true, # Upload to GCS when complete
|
||||
"gcs_path": "uploads/job123/audio/" # Required if upload_to_gcs
|
||||
}
|
||||
Response:
|
||||
{
|
||||
"download_id": "dl_xyz789",
|
||||
"status": "queued"
|
||||
}
|
||||
|
||||
# Check download status
|
||||
GET /download/{download_id}/status
|
||||
Response:
|
||||
{
|
||||
"download_id": "dl_xyz789",
|
||||
"status": "downloading", # queued, downloading, uploading, seeding, complete, failed
|
||||
"progress": 45.2,
|
||||
"peers": 3,
|
||||
"download_speed_kbps": 1250.5,
|
||||
"eta_seconds": 120,
|
||||
"provider": "Redacted",
|
||||
"title": "ABBA - Waterloo",
|
||||
"output_path": null, # Set when complete (local path)
|
||||
"gcs_path": null, # Set when uploaded to GCS
|
||||
"error": null
|
||||
}
|
||||
|
||||
# List all torrents (for management)
|
||||
GET /torrents
|
||||
Response:
|
||||
{
|
||||
"torrents": [
|
||||
{
|
||||
"id": 1,
|
||||
"name": "ABBA - Waterloo [FLAC]",
|
||||
"status": "seeding",
|
||||
"progress": 100.0,
|
||||
"size_bytes": 314572800,
|
||||
"uploaded_bytes": 628145600, # Seeding stats
|
||||
"ratio": 2.0,
|
||||
"added_at": "2025-12-20T10:30:00Z"
|
||||
},
|
||||
...
|
||||
],
|
||||
"total_size_bytes": 5368709120,
|
||||
"count": 15
|
||||
}
|
||||
|
||||
# Delete a torrent (manual cleanup)
|
||||
DELETE /torrents/{torrent_id}
|
||||
Query params: ?delete_data=true # Also delete downloaded files
|
||||
Response:
|
||||
{ "status": "deleted" }
|
||||
|
||||
# Trigger cleanup (free disk space)
|
||||
POST /torrents/cleanup
|
||||
Request:
|
||||
{
|
||||
"strategy": "oldest", # oldest, largest, lowest_ratio
|
||||
"target_free_gb": 10 # Keep removing until this much free
|
||||
}
|
||||
Response:
|
||||
{
|
||||
"removed_count": 3,
|
||||
"freed_bytes": 1073741824,
|
||||
"free_space_gb": 10.5
|
||||
}
|
||||
|
||||
# Health check
|
||||
GET /health
|
||||
Response:
|
||||
{
|
||||
"status": "healthy",
|
||||
"transmission": {
|
||||
"available": true,
|
||||
"version": "4.0.5",
|
||||
"active_torrents": 5
|
||||
},
|
||||
"disk": {
|
||||
"total_gb": 30,
|
||||
"used_gb": 18,
|
||||
"free_gb": 12
|
||||
},
|
||||
"providers": {
|
||||
"redacted": true,
|
||||
"ops": true,
|
||||
"youtube": true
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Torrent Downloader Changes:**
|
||||
|
||||
```python
|
||||
# flacfetch/downloaders/torrent.py
|
||||
|
||||
class TorrentDownloader(Downloader):
|
||||
def __init__(self, ..., keep_seeding: bool = False):
|
||||
"""
|
||||
Args:
|
||||
keep_seeding: If True, don't remove torrent after download.
|
||||
Used in server mode for indefinite seeding.
|
||||
"""
|
||||
self.keep_seeding = keep_seeding
|
||||
|
||||
def download(self, release, output_path, output_filename=None):
|
||||
# ... existing download logic ...
|
||||
|
||||
# After download complete:
|
||||
if self.keep_seeding:
|
||||
# Don't remove, just return the file path
|
||||
logger.info(f"Download complete, keeping torrent for seeding: {torrent.name}")
|
||||
else:
|
||||
# Original behavior: stop and remove
|
||||
self.client.stop_torrent(torrent.id)
|
||||
self.client.remove_torrent(torrent.id, delete_data=True)
|
||||
```
|
||||
|
||||
**CLI Changes:**
|
||||
|
||||
```bash
|
||||
# New serve subcommand
|
||||
flacfetch serve --port 8080 --host 0.0.0.0
|
||||
|
||||
# Or via environment
|
||||
FLACFETCH_API_PORT=8080 flacfetch serve
|
||||
```
|
||||
|
||||
### Phase 2: Infrastructure (Pulumi)
|
||||
|
||||
**Location:** `./infrastructure/__main__.py`
|
||||
|
||||
**New Resources:**
|
||||
```python
|
||||
# Static IP for flacfetch service (for tracker whitelist)
|
||||
flacfetch_ip = gcp.compute.Address(
|
||||
"flacfetch-ip",
|
||||
name="flacfetch-static-ip",
|
||||
region="us-central1",
|
||||
)
|
||||
|
||||
# Service account for flacfetch VM
|
||||
flacfetch_sa = gcp.serviceaccount.Account(
|
||||
"flacfetch-sa",
|
||||
account_id="flacfetch-service",
|
||||
display_name="Flacfetch Service Account",
|
||||
)
|
||||
|
||||
# GCS write permissions for flacfetch
|
||||
gcp.storage.BucketIAMMember(
|
||||
"flacfetch-gcs-writer",
|
||||
bucket=bucket.name,
|
||||
role="roles/storage.objectCreator",
|
||||
member=pulumi.Output.concat("serviceAccount:", flacfetch_sa.email),
|
||||
)
|
||||
|
||||
# Flacfetch VM
|
||||
flacfetch_vm = gcp.compute.Instance(
|
||||
"flacfetch-service",
|
||||
machine_type="e2-small",
|
||||
zone="us-central1-a",
|
||||
boot_disk=gcp.compute.InstanceBootDiskArgs(
|
||||
initialize_params=gcp.compute.InstanceBootDiskInitializeParamsArgs(
|
||||
image="debian-cloud/debian-12",
|
||||
size=30, # GB
|
||||
type="pd-ssd",
|
||||
),
|
||||
),
|
||||
network_interfaces=[gcp.compute.InstanceNetworkInterfaceArgs(
|
||||
network="default",
|
||||
access_configs=[gcp.compute.InstanceNetworkInterfaceAccessConfigArgs(
|
||||
nat_ip=flacfetch_ip.address, # Static IP
|
||||
)],
|
||||
)],
|
||||
service_account=gcp.compute.InstanceServiceAccountArgs(
|
||||
email=flacfetch_sa.email,
|
||||
scopes=["cloud-platform"],
|
||||
),
|
||||
metadata_startup_script=FLACFETCH_STARTUP_SCRIPT,
|
||||
tags=["flacfetch-service"],
|
||||
)
|
||||
|
||||
# Firewall for flacfetch
|
||||
flacfetch_firewall = gcp.compute.Firewall(
|
||||
"flacfetch-firewall",
|
||||
network="default",
|
||||
allows=[
|
||||
# BitTorrent peer connections
|
||||
gcp.compute.FirewallAllowArgs(protocol="tcp", ports=["51413"]),
|
||||
gcp.compute.FirewallAllowArgs(protocol="udp", ports=["51413"]),
|
||||
# Flacfetch HTTP API (auth required at app level)
|
||||
gcp.compute.FirewallAllowArgs(protocol="tcp", ports=["8080"]),
|
||||
],
|
||||
source_ranges=["0.0.0.0/0"], # Permissive, auth at app level
|
||||
target_tags=["flacfetch-service"],
|
||||
)
|
||||
|
||||
# Store API key in Secret Manager
|
||||
flacfetch_api_key = gcp.secretmanager.Secret(
|
||||
"flacfetch-api-key",
|
||||
secret_id="flacfetch-api-key",
|
||||
replication=gcp.secretmanager.SecretReplicationArgs(
|
||||
auto=gcp.secretmanager.SecretReplicationAutoArgs(),
|
||||
),
|
||||
)
|
||||
|
||||
# Export for use
|
||||
pulumi.export("flacfetch_static_ip", flacfetch_ip.address)
|
||||
pulumi.export("flacfetch_service_url", pulumi.Output.concat("http://", flacfetch_ip.address, ":8080"))
|
||||
```
|
||||
|
||||
**Startup Script:**
|
||||
```bash
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Install dependencies
|
||||
apt-get update
|
||||
apt-get install -y python3-pip python3-venv transmission-daemon ffmpeg git
|
||||
|
||||
# Configure Transmission
|
||||
systemctl stop transmission-daemon
|
||||
cat > /etc/transmission-daemon/settings.json << 'EOF'
|
||||
{
|
||||
"download-dir": "/var/lib/transmission-daemon/downloads",
|
||||
"incomplete-dir": "/var/lib/transmission-daemon/.incomplete",
|
||||
"incomplete-dir-enabled": true,
|
||||
"rpc-authentication-required": false,
|
||||
"rpc-bind-address": "127.0.0.1",
|
||||
"rpc-enabled": true,
|
||||
"rpc-port": 9091,
|
||||
"rpc-whitelist-enabled": false,
|
||||
"peer-port": 51413,
|
||||
"port-forwarding-enabled": false,
|
||||
"speed-limit-down": 0,
|
||||
"speed-limit-down-enabled": false,
|
||||
"speed-limit-up": 0,
|
||||
"speed-limit-up-enabled": false,
|
||||
"ratio-limit-enabled": false,
|
||||
"umask": 2
|
||||
}
|
||||
EOF
|
||||
systemctl start transmission-daemon
|
||||
|
||||
# Install flacfetch
|
||||
cd /opt
|
||||
git clone https://github.com/nomadkaraoke/flacfetch.git
|
||||
cd flacfetch
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install -e ".[api]"
|
||||
|
||||
# Get secrets from metadata/Secret Manager
|
||||
FLACFETCH_API_KEY=$(gcloud secrets versions access latest --secret=flacfetch-api-key)
|
||||
REDACTED_API_KEY=$(gcloud secrets versions access latest --secret=redacted-api-key)
|
||||
OPS_API_KEY=$(gcloud secrets versions access latest --secret=ops-api-key)
|
||||
GCS_BUCKET=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcs-bucket" -H "Metadata-Flavor: Google")
|
||||
|
||||
# Create systemd service
|
||||
cat > /etc/systemd/system/flacfetch.service << EOF
|
||||
[Unit]
|
||||
Description=Flacfetch HTTP API Service
|
||||
After=network.target transmission-daemon.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/opt/flacfetch
|
||||
Environment="FLACFETCH_API_KEY=${FLACFETCH_API_KEY}"
|
||||
Environment="REDACTED_API_KEY=${REDACTED_API_KEY}"
|
||||
Environment="OPS_API_KEY=${OPS_API_KEY}"
|
||||
Environment="GCS_BUCKET=${GCS_BUCKET}"
|
||||
Environment="FLACFETCH_KEEP_SEEDING=true"
|
||||
Environment="FLACFETCH_MIN_FREE_GB=5"
|
||||
ExecStart=/opt/flacfetch/venv/bin/flacfetch serve --host 0.0.0.0 --port 8080
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable flacfetch
|
||||
systemctl start flacfetch
|
||||
```
|
||||
|
||||
### Phase 3: Backend Integration
|
||||
|
||||
**New File:** `backend/services/flacfetch_client.py`
|
||||
|
||||
```python
|
||||
"""
|
||||
Client for the remote flacfetch HTTP API service.
|
||||
|
||||
Used when FLACFETCH_API_URL is configured, otherwise falls back to local flacfetch.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class FlacfetchServiceError(Exception):
|
||||
"""Error communicating with flacfetch service."""
|
||||
pass
|
||||
|
||||
|
||||
class FlacfetchClient:
|
||||
"""Client for remote flacfetch HTTP API."""
|
||||
|
||||
def __init__(self, base_url: str, api_key: str, timeout: int = 30):
|
||||
self.base_url = base_url.rstrip('/')
|
||||
self.api_key = api_key
|
||||
self.timeout = timeout
|
||||
|
||||
def _headers(self) -> Dict[str, str]:
|
||||
return {"X-API-Key": self.api_key}
|
||||
|
||||
async def health_check(self) -> Dict[str, Any]:
|
||||
"""Check if service is healthy."""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{self.base_url}/health",
|
||||
headers=self._headers(),
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
async def search(self, artist: str, title: str) -> Dict[str, Any]:
|
||||
"""Search for audio."""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.post(
|
||||
f"{self.base_url}/search",
|
||||
headers=self._headers(),
|
||||
json={"artist": artist, "title": title},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
async def download(
|
||||
self,
|
||||
search_id: str,
|
||||
result_index: int,
|
||||
output_filename: Optional[str] = None,
|
||||
gcs_path: Optional[str] = None,
|
||||
) -> str:
|
||||
"""
|
||||
Start download and return download_id.
|
||||
|
||||
If gcs_path is provided, file will be uploaded to GCS.
|
||||
"""
|
||||
async with httpx.AsyncClient() as client:
|
||||
payload = {
|
||||
"search_id": search_id,
|
||||
"result_index": result_index,
|
||||
}
|
||||
if output_filename:
|
||||
payload["output_filename"] = output_filename
|
||||
if gcs_path:
|
||||
payload["upload_to_gcs"] = True
|
||||
payload["gcs_path"] = gcs_path
|
||||
|
||||
resp = await client.post(
|
||||
f"{self.base_url}/download",
|
||||
headers=self._headers(),
|
||||
json=payload,
|
||||
timeout=self.timeout,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()["download_id"]
|
||||
|
||||
async def get_download_status(self, download_id: str) -> Dict[str, Any]:
|
||||
"""Get download status."""
|
||||
async with httpx.AsyncClient() as client:
|
||||
resp = await client.get(
|
||||
f"{self.base_url}/download/{download_id}/status",
|
||||
headers=self._headers(),
|
||||
timeout=10,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
async def wait_for_download(
|
||||
self,
|
||||
download_id: str,
|
||||
timeout: int = 600,
|
||||
poll_interval: int = 2,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Wait for download to complete.
|
||||
|
||||
Returns final status dict with output_path or gcs_path.
|
||||
Raises FlacfetchServiceError on failure or timeout.
|
||||
"""
|
||||
elapsed = 0
|
||||
while elapsed < timeout:
|
||||
status = await self.get_download_status(download_id)
|
||||
|
||||
if status["status"] == "complete":
|
||||
return status
|
||||
elif status["status"] == "failed":
|
||||
raise FlacfetchServiceError(f"Download failed: {status.get('error')}")
|
||||
|
||||
await asyncio.sleep(poll_interval)
|
||||
elapsed += poll_interval
|
||||
|
||||
raise FlacfetchServiceError(f"Download timed out after {timeout}s")
|
||||
|
||||
|
||||
# Singleton
|
||||
_client: Optional[FlacfetchClient] = None
|
||||
|
||||
|
||||
def get_flacfetch_client() -> Optional[FlacfetchClient]:
|
||||
"""Get flacfetch client if configured."""
|
||||
global _client
|
||||
if _client is None:
|
||||
from backend.config import get_settings
|
||||
settings = get_settings()
|
||||
if settings.flacfetch_api_url and settings.flacfetch_api_key:
|
||||
_client = FlacfetchClient(
|
||||
base_url=settings.flacfetch_api_url,
|
||||
api_key=settings.flacfetch_api_key,
|
||||
)
|
||||
return _client
|
||||
```
|
||||
|
||||
**Modify:** `backend/services/audio_search_service.py` to use remote client when available.
|
||||
|
||||
**Modify:** `backend/config.py` to add new settings:
|
||||
```python
|
||||
flacfetch_api_url: Optional[str] = None # e.g., http://10.0.0.5:8080
|
||||
flacfetch_api_key: Optional[str] = None
|
||||
```
|
||||
|
||||
### Phase 4: Monitoring & Reliability
|
||||
|
||||
**Tasks:**
|
||||
- [ ] Add Cloud Monitoring alert for VM CPU/memory
|
||||
- [ ] Add alert for disk space < 5GB
|
||||
- [ ] Add alert for flacfetch service down (health check fails)
|
||||
- [ ] Configure Cloud Logging export from VM
|
||||
- [ ] Add Grafana dashboard (optional)
|
||||
|
||||
**Automatic Disk Cleanup:**
|
||||
The disk manager runs as a background task in the flacfetch API service:
|
||||
```python
|
||||
# Check every 5 minutes, cleanup if free space < threshold
|
||||
async def background_cleanup_task():
|
||||
while True:
|
||||
await asyncio.sleep(300)
|
||||
free_gb = get_free_space_gb()
|
||||
if free_gb < MIN_FREE_GB:
|
||||
cleanup_oldest_torrents(target_free_gb=MIN_FREE_GB + 5)
|
||||
```
|
||||
|
||||
### Phase 5: Testing & Documentation
|
||||
|
||||
**Testing:**
|
||||
- [ ] Unit tests for flacfetch API endpoints
|
||||
- [ ] Unit tests for disk manager
|
||||
- [ ] Integration test: search → download → GCS upload
|
||||
- [ ] Mock tests for karaoke-gen backend client
|
||||
|
||||
**Documentation:**
|
||||
- [ ] Update `docs/01-reference/ARCHITECTURE.md` with flacfetch service
|
||||
- [ ] Add `flacfetch/README.md` HTTP API section
|
||||
- [ ] Document deployment process
|
||||
- [ ] Document environment variables
|
||||
|
||||
## Environment Variables
|
||||
|
||||
### Flacfetch Service (VM)
|
||||
```bash
|
||||
# API Configuration
|
||||
FLACFETCH_API_KEY=<secret> # Required for API auth
|
||||
FLACFETCH_API_PORT=8080 # Default: 8080
|
||||
FLACFETCH_API_HOST=0.0.0.0 # Default: 0.0.0.0
|
||||
|
||||
# Tracker API Keys
|
||||
REDACTED_API_KEY=<key> # For Redacted provider
|
||||
OPS_API_KEY=<key> # For OPS provider
|
||||
|
||||
# GCS Upload (optional)
|
||||
GCS_BUCKET=karaoke-uploads # For uploading completed downloads
|
||||
GOOGLE_CLOUD_PROJECT=nomadkaraoke
|
||||
|
||||
# Transmission
|
||||
TRANSMISSION_HOST=localhost # Default: localhost
|
||||
TRANSMISSION_PORT=9091 # Default: 9091
|
||||
|
||||
# Behavior
|
||||
FLACFETCH_KEEP_SEEDING=true # Keep torrents seeding after download
|
||||
FLACFETCH_MIN_FREE_GB=5 # Minimum free disk space before cleanup
|
||||
FLACFETCH_DOWNLOAD_DIR=/var/lib/transmission-daemon/downloads
|
||||
```
|
||||
|
||||
### Cloud Run Backend
|
||||
```bash
|
||||
# New variables
|
||||
FLACFETCH_API_URL=http://10.128.0.5:8080 # Internal IP of flacfetch VM
|
||||
FLACFETCH_API_KEY=<same-secret>
|
||||
```
|
||||
|
||||
## Security Considerations
|
||||
|
||||
1. **API Authentication**: Static API key in `X-API-Key` header
|
||||
2. **Firewall**: Permissive for future web UI, auth enforced at application level
|
||||
3. **GCS**: Service account with `storage.objectCreator` role only
|
||||
4. **Transmission**: RPC bound to localhost only, no external access
|
||||
5. **Static IP**: Can be whitelisted on tracker accounts
|
||||
|
||||
## Rollback Plan
|
||||
|
||||
If flacfetch service fails:
|
||||
1. Set `FLACFETCH_API_URL=""` in Cloud Run (disables remote service)
|
||||
2. Backend falls back to local flacfetch (YouTube only in Cloud Run)
|
||||
3. Users see warning that lossless downloads may be slower/unavailable
|
||||
|
||||
## Cost Estimate
|
||||
|
||||
| Resource | Monthly Cost |
|
||||
|----------|--------------|
|
||||
| e2-small VM | ~$12 |
|
||||
| Static IP | ~$3 |
|
||||
| 30GB SSD | ~$5 |
|
||||
| Network egress | ~$2-5 |
|
||||
| **Total** | **~$22-25/month** |
|
||||
|
||||
## Success Criteria
|
||||
|
||||
- [ ] Flacfetch HTTP API accepts search requests and returns results
|
||||
- [ ] Downloads complete successfully for Redacted/OPS sources
|
||||
- [ ] Downloaded files upload to GCS automatically
|
||||
- [ ] Torrents continue seeding after download
|
||||
- [ ] Disk cleanup triggers automatically when space is low
|
||||
- [ ] Remote CLI shows download progress from flacfetch service
|
||||
- [ ] Fallback to local flacfetch works when service unavailable
|
||||
- [ ] Static IP can be whitelisted on tracker accounts
|
||||
|
||||
## Timeline
|
||||
|
||||
| Phase | Duration | Dependencies |
|
||||
|-------|----------|--------------|
|
||||
| Phase 1: Flacfetch HTTP API | 1.5 days | None |
|
||||
| Phase 2: Infrastructure | 0.5 day | Phase 1 |
|
||||
| Phase 3: Backend Integration | 0.5 day | Phase 1, 2 |
|
||||
| Phase 4: Monitoring | 0.5 day | Phase 2 |
|
||||
| Phase 5: Testing & Docs | 0.5 day | Phase 1-3 |
|
||||
| **Total** | **3.5 days** | |
|
||||
|
||||
## References
|
||||
|
||||
- [Transmission RPC Spec](https://github.com/transmission/transmission/blob/main/docs/rpc-spec.md)
|
||||
- [transmission-rpc Python library](https://github.com/Trim21/transmission-rpc)
|
||||
- [FastAPI Documentation](https://fastapi.tiangolo.com/)
|
||||
- [flacfetch source code](./flacfetch/)
|
||||
Submodule
+1
Submodule flacfetch added at 35ae8994b8.
@@ -94,6 +94,14 @@ echo -n "your-genius-key" | \
|
||||
# Add Audio Separator API URL
|
||||
echo -n "https://your-modal-url" | \
|
||||
gcloud secrets versions add audio-separator-api-url --data-file=-
|
||||
|
||||
# Add Redacted API key (for flacfetch audio search - private tracker)
|
||||
echo -n "your-redacted-api-key" | \
|
||||
gcloud secrets versions add redacted-api-key --data-file=-
|
||||
|
||||
# Add OPS API key (for flacfetch audio search - private tracker)
|
||||
echo -n "your-ops-api-key" | \
|
||||
gcloud secrets versions add ops-api-key --data-file=-
|
||||
```
|
||||
|
||||
Or use Pulumi config secrets:
|
||||
|
||||
@@ -428,6 +428,24 @@ audio_separator_secret = secretmanager.Secret(
|
||||
),
|
||||
)
|
||||
|
||||
# Redacted API Key Secret (for flacfetch audio search)
|
||||
redacted_api_key_secret = secretmanager.Secret(
|
||||
"redacted-api-key",
|
||||
secret_id="redacted-api-key",
|
||||
replication=secretmanager.SecretReplicationArgs(
|
||||
auto=secretmanager.SecretReplicationAutoArgs(),
|
||||
),
|
||||
)
|
||||
|
||||
# OPS API Key Secret (for flacfetch audio search)
|
||||
ops_api_key_secret = secretmanager.Secret(
|
||||
"ops-api-key",
|
||||
secret_id="ops-api-key",
|
||||
replication=secretmanager.SecretReplicationArgs(
|
||||
auto=secretmanager.SecretReplicationAutoArgs(),
|
||||
),
|
||||
)
|
||||
|
||||
# Create Cloud Run Domain Mapping
|
||||
# Maps api.nomadkaraoke.com to the karaoke-backend service
|
||||
domain_mapping = cloudrun.DomainMapping(
|
||||
@@ -479,3 +497,244 @@ domain_mapping.statuses.apply(
|
||||
})
|
||||
)
|
||||
|
||||
# ==================== Flacfetch Service (Torrent Downloads) ====================
|
||||
# A dedicated VM running flacfetch with Transmission daemon for BitTorrent downloads.
|
||||
# This is required because Cloud Run doesn't support inbound peer connections for torrents.
|
||||
# The VM has a static IP for tracker whitelist and runs indefinitely for seeding.
|
||||
|
||||
# Static IP for flacfetch (for tracker account whitelist)
|
||||
flacfetch_ip = gcp.compute.Address(
|
||||
"flacfetch-ip",
|
||||
name="flacfetch-static-ip",
|
||||
region="us-central1",
|
||||
address_type="EXTERNAL",
|
||||
description="Static IP for flacfetch service (torrent downloads)",
|
||||
)
|
||||
|
||||
# Service account for flacfetch VM
|
||||
flacfetch_sa = serviceaccount.Account(
|
||||
"flacfetch-sa",
|
||||
account_id="flacfetch-service",
|
||||
display_name="Flacfetch Service Account",
|
||||
description="Service account for flacfetch torrent/audio download VM",
|
||||
)
|
||||
|
||||
# Grant flacfetch service account GCS write permissions
|
||||
flacfetch_storage_iam = gcp.storage.BucketIAMMember(
|
||||
"flacfetch-storage-writer",
|
||||
bucket=bucket.name,
|
||||
role="roles/storage.objectCreator",
|
||||
member=flacfetch_sa.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant flacfetch service account GCS read permissions (for uploads folder)
|
||||
flacfetch_storage_reader_iam = gcp.storage.BucketIAMMember(
|
||||
"flacfetch-storage-reader",
|
||||
bucket=bucket.name,
|
||||
role="roles/storage.objectViewer",
|
||||
member=flacfetch_sa.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Grant flacfetch service account Secret Manager access
|
||||
flacfetch_secrets_iam = gcp.projects.IAMMember(
|
||||
"flacfetch-secrets-access",
|
||||
project=project_id,
|
||||
role="roles/secretmanager.secretAccessor",
|
||||
member=flacfetch_sa.email.apply(lambda email: f"serviceAccount:{email}"),
|
||||
)
|
||||
|
||||
# Flacfetch API key secret (for authenticating API requests)
|
||||
flacfetch_api_key_secret = secretmanager.Secret(
|
||||
"flacfetch-api-key",
|
||||
secret_id="flacfetch-api-key",
|
||||
replication=secretmanager.SecretReplicationArgs(
|
||||
auto=secretmanager.SecretReplicationAutoArgs(),
|
||||
),
|
||||
)
|
||||
|
||||
# Flacfetch API URL secret (the static IP address of the flacfetch VM)
|
||||
# This is stored as a secret to avoid committing infrastructure IPs to the repo
|
||||
flacfetch_api_url_secret = secretmanager.Secret(
|
||||
"flacfetch-api-url",
|
||||
secret_id="flacfetch-api-url",
|
||||
replication=secretmanager.SecretReplicationArgs(
|
||||
auto=secretmanager.SecretReplicationAutoArgs(),
|
||||
),
|
||||
)
|
||||
|
||||
# Startup script for the flacfetch VM
|
||||
# This installs and configures Transmission daemon and the flacfetch service
|
||||
FLACFETCH_STARTUP_SCRIPT = """#!/bin/bash
|
||||
set -e
|
||||
|
||||
# Log to a file for debugging
|
||||
exec > >(tee /var/log/flacfetch-startup.log) 2>&1
|
||||
echo "Starting flacfetch setup at $(date)"
|
||||
|
||||
# Install dependencies
|
||||
apt-get update
|
||||
apt-get install -y python3-pip python3-venv transmission-daemon ffmpeg git curl
|
||||
|
||||
# Configure Transmission
|
||||
echo "Configuring Transmission daemon..."
|
||||
systemctl stop transmission-daemon || true
|
||||
|
||||
# Create transmission directories
|
||||
mkdir -p /var/lib/transmission-daemon/downloads
|
||||
mkdir -p /var/lib/transmission-daemon/.config/transmission-daemon
|
||||
chown -R debian-transmission:debian-transmission /var/lib/transmission-daemon
|
||||
|
||||
# Configure transmission settings
|
||||
cat > /var/lib/transmission-daemon/.config/transmission-daemon/settings.json << 'SETTINGS'
|
||||
{
|
||||
"download-dir": "/var/lib/transmission-daemon/downloads",
|
||||
"incomplete-dir": "/var/lib/transmission-daemon/.incomplete",
|
||||
"incomplete-dir-enabled": true,
|
||||
"rpc-authentication-required": false,
|
||||
"rpc-bind-address": "127.0.0.1",
|
||||
"rpc-enabled": true,
|
||||
"rpc-port": 9091,
|
||||
"rpc-whitelist-enabled": false,
|
||||
"peer-port": 51413,
|
||||
"port-forwarding-enabled": false,
|
||||
"speed-limit-down": 0,
|
||||
"speed-limit-down-enabled": false,
|
||||
"speed-limit-up": 0,
|
||||
"speed-limit-up-enabled": false,
|
||||
"ratio-limit-enabled": false,
|
||||
"umask": 2,
|
||||
"encryption": 1
|
||||
}
|
||||
SETTINGS
|
||||
|
||||
chown debian-transmission:debian-transmission /var/lib/transmission-daemon/.config/transmission-daemon/settings.json
|
||||
systemctl start transmission-daemon
|
||||
echo "Transmission daemon started"
|
||||
|
||||
# Install flacfetch
|
||||
echo "Installing flacfetch..."
|
||||
cd /opt
|
||||
|
||||
# Clone flacfetch (we'll use the repo from GitHub)
|
||||
if [ ! -d "flacfetch" ]; then
|
||||
git clone https://github.com/nomadkaraoke/flacfetch.git
|
||||
fi
|
||||
cd flacfetch
|
||||
|
||||
# Create and activate virtual environment
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
|
||||
# Install flacfetch with API dependencies
|
||||
pip install -e ".[api]"
|
||||
|
||||
# Get secrets from Secret Manager
|
||||
echo "Fetching secrets from Secret Manager..."
|
||||
FLACFETCH_API_KEY=$(gcloud secrets versions access latest --secret=flacfetch-api-key 2>/dev/null || echo "")
|
||||
REDACTED_API_KEY=$(gcloud secrets versions access latest --secret=redacted-api-key 2>/dev/null || echo "")
|
||||
OPS_API_KEY=$(gcloud secrets versions access latest --secret=ops-api-key 2>/dev/null || echo "")
|
||||
|
||||
# Get bucket name from project metadata
|
||||
GCS_BUCKET=$(gcloud compute project-info describe --format='value(commonInstanceMetadata.items.gcs-bucket)' 2>/dev/null || echo "")
|
||||
if [ -z "$GCS_BUCKET" ]; then
|
||||
# Fallback: construct from project ID
|
||||
PROJECT_ID=$(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google")
|
||||
GCS_BUCKET="karaoke-gen-storage-${PROJECT_ID}"
|
||||
fi
|
||||
|
||||
echo "Using GCS bucket: $GCS_BUCKET"
|
||||
|
||||
# Create systemd service for flacfetch
|
||||
echo "Creating systemd service..."
|
||||
cat > /etc/systemd/system/flacfetch.service << SYSTEMD
|
||||
[Unit]
|
||||
Description=Flacfetch HTTP API Service
|
||||
After=network.target transmission-daemon.service
|
||||
Requires=transmission-daemon.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
WorkingDirectory=/opt/flacfetch
|
||||
Environment="FLACFETCH_API_KEY=${FLACFETCH_API_KEY}"
|
||||
Environment="REDACTED_API_KEY=${REDACTED_API_KEY}"
|
||||
Environment="OPS_API_KEY=${OPS_API_KEY}"
|
||||
Environment="GCS_BUCKET=${GCS_BUCKET}"
|
||||
Environment="FLACFETCH_KEEP_SEEDING=true"
|
||||
Environment="FLACFETCH_MIN_FREE_GB=5"
|
||||
Environment="FLACFETCH_DOWNLOAD_DIR=/var/lib/transmission-daemon/downloads"
|
||||
Environment="TRANSMISSION_HOST=localhost"
|
||||
Environment="TRANSMISSION_PORT=9091"
|
||||
ExecStart=/opt/flacfetch/venv/bin/flacfetch serve --host 0.0.0.0 --port 8080
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
SYSTEMD
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable flacfetch
|
||||
systemctl start flacfetch
|
||||
|
||||
echo "Flacfetch service started at $(date)"
|
||||
echo "Setup complete!"
|
||||
"""
|
||||
|
||||
# Flacfetch VM instance
|
||||
flacfetch_vm = gcp.compute.Instance(
|
||||
"flacfetch-service",
|
||||
name="flacfetch-service",
|
||||
machine_type="e2-small", # 0.5 vCPU, 2GB RAM - sufficient for I/O-bound workload
|
||||
zone="us-central1-a",
|
||||
boot_disk=gcp.compute.InstanceBootDiskArgs(
|
||||
initialize_params=gcp.compute.InstanceBootDiskInitializeParamsArgs(
|
||||
image="debian-cloud/debian-12",
|
||||
size=30, # GB - for torrent storage + headroom
|
||||
type="pd-ssd", # SSD for faster I/O
|
||||
),
|
||||
),
|
||||
network_interfaces=[gcp.compute.InstanceNetworkInterfaceArgs(
|
||||
network="default",
|
||||
access_configs=[gcp.compute.InstanceNetworkInterfaceAccessConfigArgs(
|
||||
nat_ip=flacfetch_ip.address, # Static IP
|
||||
)],
|
||||
)],
|
||||
service_account=gcp.compute.InstanceServiceAccountArgs(
|
||||
email=flacfetch_sa.email,
|
||||
scopes=["cloud-platform"],
|
||||
),
|
||||
metadata_startup_script=FLACFETCH_STARTUP_SCRIPT,
|
||||
metadata={
|
||||
"gcs-bucket": bucket.name, # Pass bucket name to startup script
|
||||
},
|
||||
tags=["flacfetch-service"],
|
||||
allow_stopping_for_update=True,
|
||||
)
|
||||
|
||||
# Firewall rule for flacfetch
|
||||
# - Port 51413 TCP/UDP: BitTorrent peer connections
|
||||
# - Port 8080 TCP: Flacfetch HTTP API (auth required at app level)
|
||||
flacfetch_firewall = gcp.compute.Firewall(
|
||||
"flacfetch-firewall",
|
||||
name="flacfetch-firewall",
|
||||
network="default",
|
||||
allows=[
|
||||
# BitTorrent peer connections
|
||||
gcp.compute.FirewallAllowArgs(protocol="tcp", ports=["51413"]),
|
||||
gcp.compute.FirewallAllowArgs(protocol="udp", ports=["51413"]),
|
||||
# Flacfetch HTTP API (auth required at app level)
|
||||
gcp.compute.FirewallAllowArgs(protocol="tcp", ports=["8080"]),
|
||||
],
|
||||
source_ranges=["0.0.0.0/0"], # Permissive, auth enforced at app level
|
||||
target_tags=["flacfetch-service"],
|
||||
description="Firewall rules for flacfetch torrent/API service",
|
||||
)
|
||||
|
||||
# Export flacfetch values
|
||||
pulumi.export("flacfetch_static_ip", flacfetch_ip.address)
|
||||
pulumi.export("flacfetch_service_url", flacfetch_ip.address.apply(
|
||||
lambda ip: f"http://{ip}:8080"
|
||||
))
|
||||
pulumi.export("flacfetch_service_account", flacfetch_sa.email)
|
||||
|
||||
+32
-1
@@ -4,4 +4,35 @@ import warnings
|
||||
warnings.filterwarnings("ignore", category=SyntaxWarning, module="pydub.*")
|
||||
warnings.filterwarnings("ignore", category=SyntaxWarning, module="syrics.*")
|
||||
|
||||
from .karaoke_gen import KaraokePrep
|
||||
# Lazy imports to avoid loading heavy dependencies when not needed.
|
||||
# KaraokePrep has many dependencies (lyrics_transcriber, etc.) which may not
|
||||
# be available in all contexts (e.g., backend which only needs audio_fetcher).
|
||||
#
|
||||
# Use explicit imports instead:
|
||||
# from karaoke_gen.karaoke_gen import KaraokePrep
|
||||
# from karaoke_gen.audio_fetcher import FlacFetcher, AudioSearchResult, ...
|
||||
|
||||
__all__ = [
|
||||
"KaraokePrep",
|
||||
# Audio fetcher
|
||||
"FlacFetcher",
|
||||
"AudioSearchResult",
|
||||
"AudioFetchResult",
|
||||
"AudioFetcherError",
|
||||
"NoResultsError",
|
||||
"DownloadError",
|
||||
"UserCancelledError",
|
||||
]
|
||||
|
||||
|
||||
def __getattr__(name):
|
||||
"""Lazy import for heavy modules."""
|
||||
if name == "KaraokePrep":
|
||||
from .karaoke_gen import KaraokePrep
|
||||
return KaraokePrep
|
||||
elif name in ("FlacFetcher", "AudioSearchResult", "AudioFetchResult",
|
||||
"AudioFetcherError", "NoResultsError", "DownloadError",
|
||||
"UserCancelledError"):
|
||||
from . import audio_fetcher
|
||||
return getattr(audio_fetcher, name)
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
+313
-53
@@ -13,8 +13,8 @@ import tempfile
|
||||
import threading
|
||||
from abc import ABC, abstractmethod
|
||||
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional
|
||||
from dataclasses import dataclass, asdict, field
|
||||
from typing import List, Optional, Dict, Any
|
||||
|
||||
# Global flag to track if user requested cancellation via Ctrl+C
|
||||
_interrupt_requested = False
|
||||
@@ -22,7 +22,14 @@ _interrupt_requested = False
|
||||
|
||||
@dataclass
|
||||
class AudioSearchResult:
|
||||
"""Represents a single search result for audio."""
|
||||
"""Represents a single search result for audio.
|
||||
|
||||
Used by both local CLI and cloud backend. Supports serialization
|
||||
for Firestore storage via to_dict()/from_dict().
|
||||
|
||||
For rich display, this class can serialize the full flacfetch Release
|
||||
data so remote CLIs can use flacfetch's shared display functions.
|
||||
"""
|
||||
|
||||
title: str
|
||||
artist: str
|
||||
@@ -31,13 +38,75 @@ class AudioSearchResult:
|
||||
duration: Optional[int] = None # Duration in seconds
|
||||
quality: Optional[str] = None # e.g., "FLAC", "320kbps", etc.
|
||||
source_id: Optional[str] = None # Unique ID from the source
|
||||
# Raw result object from the provider (for download)
|
||||
raw_result: Optional[object] = None
|
||||
index: int = 0 # Index in the results list (for API selection)
|
||||
seeders: Optional[int] = None # Number of seeders (for torrent sources)
|
||||
target_file: Optional[str] = None # Target filename in the release
|
||||
# Raw result object from the provider (for download) - not serialized
|
||||
raw_result: Optional[object] = field(default=None, repr=False)
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dict for JSON/Firestore serialization.
|
||||
|
||||
Includes full flacfetch Release data if available, enabling
|
||||
remote CLIs to use flacfetch's shared display functions.
|
||||
"""
|
||||
result = {
|
||||
"title": self.title,
|
||||
"artist": self.artist,
|
||||
"url": self.url,
|
||||
"provider": self.provider,
|
||||
"duration": self.duration,
|
||||
"quality": self.quality,
|
||||
"source_id": self.source_id,
|
||||
"index": self.index,
|
||||
"seeders": self.seeders,
|
||||
"target_file": self.target_file,
|
||||
}
|
||||
|
||||
# If we have a raw_result (flacfetch Release), include its full data
|
||||
# This enables rich display on the remote CLI
|
||||
if self.raw_result:
|
||||
try:
|
||||
release_dict = self.raw_result.to_dict()
|
||||
# Merge Release fields into result (they may override basic fields)
|
||||
for key in ['year', 'label', 'edition_info', 'release_type', 'channel',
|
||||
'view_count', 'size_bytes', 'target_file_size', 'track_pattern',
|
||||
'match_score', 'formatted_size', 'formatted_duration',
|
||||
'formatted_views', 'is_lossless', 'quality_str', 'quality']:
|
||||
if key in release_dict:
|
||||
# Use 'quality_data' for the quality dict to avoid confusion with quality string
|
||||
result_key = 'quality_data' if key == 'quality' else key
|
||||
result[result_key] = release_dict[key]
|
||||
except AttributeError:
|
||||
pass # raw_result doesn't have to_dict() method
|
||||
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AudioSearchResult":
|
||||
"""Create from dict (e.g., from Firestore)."""
|
||||
return cls(
|
||||
title=data.get("title", ""),
|
||||
artist=data.get("artist", ""),
|
||||
url=data.get("url", ""),
|
||||
provider=data.get("provider", "Unknown"),
|
||||
duration=data.get("duration"),
|
||||
quality=data.get("quality"),
|
||||
source_id=data.get("source_id"),
|
||||
index=data.get("index", 0),
|
||||
seeders=data.get("seeders"),
|
||||
target_file=data.get("target_file"),
|
||||
raw_result=None, # Not stored in serialized form
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AudioFetchResult:
|
||||
"""Result of an audio fetch operation."""
|
||||
"""Result of an audio fetch operation.
|
||||
|
||||
Used by both local CLI and cloud backend. Supports serialization
|
||||
for Firestore storage via to_dict()/from_dict().
|
||||
"""
|
||||
|
||||
filepath: str
|
||||
artist: str
|
||||
@@ -45,6 +114,22 @@ class AudioFetchResult:
|
||||
provider: str
|
||||
duration: Optional[int] = None
|
||||
quality: Optional[str] = None
|
||||
|
||||
def to_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to dict for JSON/Firestore serialization."""
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: Dict[str, Any]) -> "AudioFetchResult":
|
||||
"""Create from dict (e.g., from Firestore)."""
|
||||
return cls(
|
||||
filepath=data.get("filepath", ""),
|
||||
artist=data.get("artist", ""),
|
||||
title=data.get("title", ""),
|
||||
provider=data.get("provider", "Unknown"),
|
||||
duration=data.get("duration"),
|
||||
quality=data.get("quality"),
|
||||
)
|
||||
|
||||
|
||||
class AudioFetcherError(Exception):
|
||||
@@ -160,6 +245,8 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
|
||||
This provides access to multiple audio sources including private music trackers
|
||||
and YouTube, with intelligent prioritization of high-quality sources.
|
||||
|
||||
Also exported as FlacFetcher for shorter name.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -183,6 +270,47 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
self._ops_api_key = ops_api_key or os.environ.get("OPS_API_KEY")
|
||||
self._provider_priority = provider_priority
|
||||
self._manager = None
|
||||
self._transmission_available = None # Cached result of Transmission check
|
||||
|
||||
def _check_transmission_available(self) -> bool:
|
||||
"""
|
||||
Check if Transmission daemon is available for torrent downloads.
|
||||
|
||||
This prevents adding tracker providers (Redacted/OPS) when Transmission
|
||||
isn't running, which would result in search results that can't be downloaded.
|
||||
|
||||
Returns:
|
||||
True if Transmission is available and responsive, False otherwise.
|
||||
"""
|
||||
if self._transmission_available is not None:
|
||||
self.logger.info(f"[Transmission] Using cached status: available={self._transmission_available}")
|
||||
return self._transmission_available
|
||||
|
||||
host = os.environ.get("TRANSMISSION_HOST", "localhost")
|
||||
port = int(os.environ.get("TRANSMISSION_PORT", "9091"))
|
||||
self.logger.info(f"[Transmission] Checking availability at {host}:{port}")
|
||||
|
||||
try:
|
||||
import transmission_rpc
|
||||
self.logger.info(f"[Transmission] transmission_rpc imported successfully")
|
||||
|
||||
client = transmission_rpc.Client(host=host, port=port, timeout=5)
|
||||
self.logger.info(f"[Transmission] Client created, calling session_stats()...")
|
||||
|
||||
# Simple test to verify connection works
|
||||
stats = client.session_stats()
|
||||
self.logger.info(f"[Transmission] Connected! Download dir: {getattr(stats, 'download_dir', 'unknown')}")
|
||||
|
||||
self._transmission_available = True
|
||||
except ImportError as e:
|
||||
self._transmission_available = False
|
||||
self.logger.warning(f"[Transmission] transmission_rpc not installed: {e}")
|
||||
except Exception as e:
|
||||
self._transmission_available = False
|
||||
self.logger.warning(f"[Transmission] Connection failed to {host}:{port}: {type(e).__name__}: {e}")
|
||||
|
||||
self.logger.info(f"[Transmission] Final status: available={self._transmission_available}")
|
||||
return self._transmission_available
|
||||
|
||||
def _get_manager(self):
|
||||
"""Lazily initialize and return the FetchManager."""
|
||||
@@ -193,30 +321,46 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
from flacfetch.downloaders.youtube import YoutubeDownloader
|
||||
|
||||
# Try to import TorrentDownloader (has optional dependencies)
|
||||
TorrentDownloader = None
|
||||
try:
|
||||
from flacfetch.downloaders.torrent import TorrentDownloader
|
||||
except ImportError:
|
||||
TorrentDownloader = None
|
||||
self.logger.debug("TorrentDownloader not available (missing dependencies)")
|
||||
|
||||
self._manager = FetchManager()
|
||||
|
||||
# Only add tracker providers if we can actually download from them
|
||||
# This requires both TorrentDownloader and a running Transmission daemon
|
||||
has_torrent_downloader = TorrentDownloader is not None
|
||||
transmission_available = self._check_transmission_available()
|
||||
can_use_trackers = has_torrent_downloader and transmission_available
|
||||
|
||||
self.logger.info(
|
||||
f"[FlacFetcher] Provider setup: TorrentDownloader={has_torrent_downloader}, "
|
||||
f"Transmission={transmission_available}, can_use_trackers={can_use_trackers}"
|
||||
)
|
||||
|
||||
if not can_use_trackers and (self._redacted_api_key or self._ops_api_key):
|
||||
self.logger.warning(
|
||||
"[FlacFetcher] Tracker providers (Redacted/OPS) DISABLED: "
|
||||
f"TorrentDownloader={has_torrent_downloader}, Transmission={transmission_available}. "
|
||||
"Only YouTube sources will be used."
|
||||
)
|
||||
|
||||
# Add providers and downloaders based on available API keys
|
||||
if self._redacted_api_key:
|
||||
if self._redacted_api_key and can_use_trackers:
|
||||
from flacfetch.providers.redacted import RedactedProvider
|
||||
|
||||
self._manager.add_provider(RedactedProvider(api_key=self._redacted_api_key))
|
||||
if TorrentDownloader:
|
||||
self._manager.register_downloader("Redacted", TorrentDownloader())
|
||||
self.logger.debug("Added Redacted provider")
|
||||
self._manager.register_downloader("Redacted", TorrentDownloader())
|
||||
self.logger.info("[FlacFetcher] Added Redacted provider with TorrentDownloader")
|
||||
|
||||
if self._ops_api_key:
|
||||
if self._ops_api_key and can_use_trackers:
|
||||
from flacfetch.providers.ops import OPSProvider
|
||||
|
||||
self._manager.add_provider(OPSProvider(api_key=self._ops_api_key))
|
||||
if TorrentDownloader:
|
||||
self._manager.register_downloader("OPS", TorrentDownloader())
|
||||
self.logger.debug("Added OPS provider")
|
||||
self._manager.register_downloader("OPS", TorrentDownloader())
|
||||
self.logger.info("[FlacFetcher] Added OPS provider with TorrentDownloader")
|
||||
|
||||
# Always add YouTube as a fallback provider with its downloader
|
||||
self._manager.add_provider(YoutubeProvider())
|
||||
@@ -252,7 +396,7 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
|
||||
# Convert to our AudioSearchResult format
|
||||
search_results = []
|
||||
for result in results:
|
||||
for i, result in enumerate(results):
|
||||
# Get quality as string if it's a Quality object
|
||||
quality = getattr(result, "quality", None)
|
||||
quality_str = str(quality) if quality else None
|
||||
@@ -266,6 +410,9 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
duration=getattr(result, "duration_seconds", None),
|
||||
quality=quality_str,
|
||||
source_id=getattr(result, "info_hash", None),
|
||||
index=i, # Set index for API selection
|
||||
seeders=getattr(result, "seeders", None),
|
||||
target_file=getattr(result, "target_file", None),
|
||||
raw_result=result,
|
||||
)
|
||||
)
|
||||
@@ -329,6 +476,40 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
except Exception as e:
|
||||
raise DownloadError(f"Failed to download {result.artist} - {result.title}: {e}") from e
|
||||
|
||||
def select_best(self, results: List[AudioSearchResult]) -> int:
|
||||
"""
|
||||
Select the best result from a list of search results.
|
||||
|
||||
Uses flacfetch's built-in quality ranking to determine the best source.
|
||||
This is useful for automated/non-interactive usage.
|
||||
|
||||
Args:
|
||||
results: List of AudioSearchResult objects from search()
|
||||
|
||||
Returns:
|
||||
Index of the best result in the list
|
||||
"""
|
||||
if not results:
|
||||
return 0
|
||||
|
||||
manager = self._get_manager()
|
||||
|
||||
# Get raw results that have raw_result set
|
||||
raw_results = [r.raw_result for r in results if r.raw_result is not None]
|
||||
|
||||
if raw_results:
|
||||
try:
|
||||
best = manager.select_best(raw_results)
|
||||
# Find index of best result
|
||||
for i, r in enumerate(results):
|
||||
if r.raw_result == best:
|
||||
return i
|
||||
except Exception as e:
|
||||
self.logger.warning(f"select_best failed, using first result: {e}")
|
||||
|
||||
# Fallback: return first result
|
||||
return 0
|
||||
|
||||
def search_and_download(
|
||||
self,
|
||||
artist: str,
|
||||
@@ -396,13 +577,15 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
self.logger.info(f"Downloading from {getattr(selected, 'source_name', 'Unknown')}...")
|
||||
|
||||
try:
|
||||
filepath = manager.download(
|
||||
# Use interruptible download so Ctrl+C works during torrent downloads
|
||||
filepath = self._interruptible_download(
|
||||
manager,
|
||||
selected,
|
||||
output_path=output_dir,
|
||||
output_filename=output_filename,
|
||||
)
|
||||
|
||||
if filepath is None:
|
||||
if not filepath:
|
||||
raise DownloadError(f"Download returned no file path for: {artist} - {title}")
|
||||
|
||||
self.logger.info(f"Downloaded to: {filepath}")
|
||||
@@ -488,6 +671,111 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
signal.signal(signal.SIGINT, original_handler)
|
||||
_interrupt_requested = False
|
||||
|
||||
def _interruptible_download(self, manager, selected, output_path: str, output_filename: str) -> str:
|
||||
"""
|
||||
Run download in a way that can be interrupted by Ctrl+C.
|
||||
|
||||
The flacfetch/transmission download is a blocking operation that doesn't
|
||||
respond to SIGINT while running (especially for torrent downloads).
|
||||
This method runs it in a background thread and periodically checks for interrupts.
|
||||
|
||||
Args:
|
||||
manager: The FetchManager instance
|
||||
selected: The selected result to download
|
||||
output_path: Directory to save the file
|
||||
output_filename: Filename to save as
|
||||
|
||||
Returns:
|
||||
Path to the downloaded file
|
||||
|
||||
Raises:
|
||||
UserCancelledError: If user presses Ctrl+C during download
|
||||
DownloadError: If download fails
|
||||
"""
|
||||
global _interrupt_requested
|
||||
_interrupt_requested = False
|
||||
result_container = {"filepath": None, "error": None}
|
||||
was_cancelled = False
|
||||
|
||||
def do_download():
|
||||
try:
|
||||
result_container["filepath"] = manager.download(
|
||||
selected,
|
||||
output_path=output_path,
|
||||
output_filename=output_filename,
|
||||
)
|
||||
except Exception as e:
|
||||
result_container["error"] = e
|
||||
|
||||
# Set up signal handler for immediate response to Ctrl+C
|
||||
original_handler = signal.getsignal(signal.SIGINT)
|
||||
|
||||
def interrupt_handler(signum, frame):
|
||||
global _interrupt_requested
|
||||
_interrupt_requested = True
|
||||
# Print immediately so user knows it was received
|
||||
print("\nCancelling download... please wait (may take a few seconds)", file=sys.stderr)
|
||||
|
||||
signal.signal(signal.SIGINT, interrupt_handler)
|
||||
|
||||
try:
|
||||
# Start download in background thread
|
||||
download_thread = threading.Thread(target=do_download, daemon=True)
|
||||
download_thread.start()
|
||||
|
||||
# Wait for completion with periodic interrupt checks
|
||||
while download_thread.is_alive():
|
||||
download_thread.join(timeout=0.2) # Check every 200ms
|
||||
if _interrupt_requested:
|
||||
was_cancelled = True
|
||||
# Clean up any pending torrents before raising
|
||||
self._cleanup_transmission_torrents(selected)
|
||||
raise UserCancelledError("Download cancelled by user (Ctrl+C)")
|
||||
|
||||
# Check for errors from the download
|
||||
if result_container["error"] is not None:
|
||||
raise result_container["error"]
|
||||
|
||||
return result_container["filepath"]
|
||||
|
||||
finally:
|
||||
# Restore original signal handler
|
||||
signal.signal(signal.SIGINT, original_handler)
|
||||
_interrupt_requested = False
|
||||
|
||||
def _cleanup_transmission_torrents(self, selected) -> None:
|
||||
"""
|
||||
Clean up any torrents in Transmission that were started for this download.
|
||||
|
||||
Called when a download is cancelled to remove incomplete torrents and their data.
|
||||
|
||||
Args:
|
||||
selected: The selected result that was being downloaded
|
||||
"""
|
||||
try:
|
||||
import transmission_rpc
|
||||
host = os.environ.get("TRANSMISSION_HOST", "localhost")
|
||||
port = int(os.environ.get("TRANSMISSION_PORT", "9091"))
|
||||
client = transmission_rpc.Client(host=host, port=port, timeout=5)
|
||||
|
||||
# Get the release name to match against torrents
|
||||
release_name = getattr(selected, 'name', None) or getattr(selected, 'title', None)
|
||||
if not release_name:
|
||||
self.logger.debug("[Transmission] No release name to match for cleanup")
|
||||
return
|
||||
|
||||
# Find and remove matching incomplete torrents
|
||||
torrents = client.get_torrents()
|
||||
for torrent in torrents:
|
||||
# Match by name similarity and incomplete status
|
||||
if torrent.progress < 100 and release_name.lower() in torrent.name.lower():
|
||||
self.logger.info(f"[Transmission] Removing cancelled torrent: {torrent.name}")
|
||||
client.remove_torrent(torrent.id, delete_data=True)
|
||||
|
||||
except Exception as e:
|
||||
# Don't fail the cancellation if cleanup fails
|
||||
self.logger.debug(f"[Transmission] Cleanup failed (non-fatal): {e}")
|
||||
|
||||
def _interactive_select(self, results: list, artist: str, title: str) -> object:
|
||||
"""
|
||||
Present search results to the user for interactive selection.
|
||||
@@ -539,41 +827,9 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
Raises:
|
||||
UserCancelledError: If user cancels selection
|
||||
"""
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"Search Results for: {artist} - {title}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
for i, result in enumerate(results, 1):
|
||||
# Use correct flacfetch attribute names
|
||||
source_name = getattr(result, "source_name", "Unknown")
|
||||
result_title = getattr(result, "title", "Unknown")
|
||||
result_artist = getattr(result, "artist", "Unknown")
|
||||
quality = getattr(result, "quality", None)
|
||||
duration_seconds = getattr(result, "duration_seconds", None)
|
||||
seeders = getattr(result, "seeders", None)
|
||||
target_file = getattr(result, "target_file", None)
|
||||
|
||||
# Format duration if available
|
||||
duration_str = ""
|
||||
if duration_seconds:
|
||||
minutes = duration_seconds // 60
|
||||
seconds = duration_seconds % 60
|
||||
duration_str = f" [{minutes}:{seconds:02d}]"
|
||||
|
||||
# Format quality - it's a Quality object with __str__
|
||||
quality_str = f" ({quality})" if quality else ""
|
||||
|
||||
# Format seeders for torrent sources
|
||||
seeders_str = f" Seeders: {seeders}" if seeders is not None else ""
|
||||
|
||||
# Format target file
|
||||
file_str = f' "{target_file}"' if target_file else ""
|
||||
|
||||
print(f" {i}. [{source_name}] {result_artist} - {result_title}{quality_str}{duration_str}{seeders_str}{file_str}")
|
||||
|
||||
print()
|
||||
print(" 0. Cancel")
|
||||
print()
|
||||
# Use flacfetch's shared display function
|
||||
from flacfetch import print_releases
|
||||
print_releases(results, target_artist=artist, use_colors=True)
|
||||
|
||||
while True:
|
||||
try:
|
||||
@@ -598,6 +854,10 @@ class FlacFetchAudioFetcher(AudioFetcher):
|
||||
raise UserCancelledError("Selection cancelled by user (Ctrl+C)")
|
||||
|
||||
|
||||
# Alias for shorter name - used by backend and other consumers
|
||||
FlacFetcher = FlacFetchAudioFetcher
|
||||
|
||||
|
||||
def create_audio_fetcher(
|
||||
logger: Optional[logging.Logger] = None,
|
||||
redacted_api_key: Optional[str] = None,
|
||||
|
||||
@@ -771,8 +771,16 @@ class AudioProcessor:
|
||||
padded_result["other_stems"] = separation_result.get("other_stems", {})
|
||||
padded_result["backing_vocals"] = separation_result.get("backing_vocals", {})
|
||||
|
||||
self.logger.info(
|
||||
f"✓ Countdown padding applied to {len(padded_result['combined_instrumentals']) + 1} instrumental file(s)"
|
||||
)
|
||||
# Count actual padded files (don't assume clean instrumental was padded)
|
||||
padded_count = (1 if padded_result["clean_instrumental"].get("instrumental") else 0) + len(padded_result["combined_instrumentals"])
|
||||
|
||||
if padded_count > 0:
|
||||
self.logger.info(
|
||||
f"✓ Countdown padding applied to {padded_count} instrumental file(s)"
|
||||
)
|
||||
else:
|
||||
self.logger.warning(
|
||||
"No instrumental files found to pad. Check that audio separation completed successfully."
|
||||
)
|
||||
|
||||
return padded_result
|
||||
@@ -47,6 +47,7 @@ class KaraokeFinalise:
|
||||
user_youtube_credentials=None, # Add support for pre-stored credentials
|
||||
server_side_mode=False, # New parameter for server-side deployment
|
||||
selected_instrumental_file=None, # Add support for pre-selected instrumental file
|
||||
countdown_padding_seconds=None, # Padding applied to vocals; instrumental must match
|
||||
):
|
||||
self.log_level = log_level
|
||||
self.log_formatter = log_formatter
|
||||
@@ -108,6 +109,7 @@ class KaraokeFinalise:
|
||||
self.user_youtube_credentials = user_youtube_credentials
|
||||
self.server_side_mode = server_side_mode
|
||||
self.selected_instrumental_file = selected_instrumental_file
|
||||
self.countdown_padding_seconds = countdown_padding_seconds
|
||||
|
||||
self.suffixes = {
|
||||
"title_mov": " (Title).mov",
|
||||
@@ -732,6 +734,32 @@ class KaraokeFinalise:
|
||||
artist, title = base_name.split(" - ", 1)
|
||||
return base_name, artist, title
|
||||
|
||||
def _pad_audio_file(self, input_audio, output_audio, padding_seconds):
|
||||
"""
|
||||
Pad an audio file by prepending silence at the beginning.
|
||||
|
||||
Uses the same ffmpeg approach as LyricsTranscriber's CountdownProcessor
|
||||
to ensure consistent padding behavior.
|
||||
|
||||
Args:
|
||||
input_audio: Path to input audio file
|
||||
output_audio: Path for the padded output file
|
||||
padding_seconds: Amount of silence to prepend (in seconds)
|
||||
"""
|
||||
self.logger.info(f"Padding audio file with {padding_seconds}s of silence")
|
||||
|
||||
# Use ffmpeg to prepend silence - this matches the approach in audio_processor.py
|
||||
# adelay filter adds delay in milliseconds
|
||||
delay_ms = int(padding_seconds * 1000)
|
||||
|
||||
ffmpeg_command = (
|
||||
f'{self.ffmpeg_base_command} -i "{input_audio}" '
|
||||
f'-af "adelay={delay_ms}|{delay_ms}" '
|
||||
f'"{output_audio}"'
|
||||
)
|
||||
|
||||
self.execute_command(ffmpeg_command, f"Padding audio with {padding_seconds}s silence")
|
||||
|
||||
def execute_command(self, command, description):
|
||||
"""Execute a shell command and log the output. For general commands (rclone, etc.)"""
|
||||
self.logger.info(f"{description}")
|
||||
@@ -776,11 +804,32 @@ class KaraokeFinalise:
|
||||
|
||||
def remux_with_instrumental(self, with_vocals_file, instrumental_audio, output_file):
|
||||
"""Remux the video with instrumental audio to create karaoke version"""
|
||||
# Safety net: If countdown padding was applied to vocals, ensure instrumental is padded too
|
||||
actual_instrumental = instrumental_audio
|
||||
if self.countdown_padding_seconds and self.countdown_padding_seconds > 0:
|
||||
# Check if the instrumental file is already padded (has "(Padded)" in name)
|
||||
if "(Padded)" not in instrumental_audio:
|
||||
self.logger.warning(
|
||||
f"Countdown padding ({self.countdown_padding_seconds}s) was applied to vocals, "
|
||||
f"but instrumental doesn't appear to be padded. Creating padded version..."
|
||||
)
|
||||
# Create a padded version of the instrumental
|
||||
base, ext = os.path.splitext(instrumental_audio)
|
||||
padded_instrumental = f"{base} (Padded){ext}"
|
||||
|
||||
if not os.path.exists(padded_instrumental):
|
||||
self._pad_audio_file(instrumental_audio, padded_instrumental, self.countdown_padding_seconds)
|
||||
self.logger.info(f"Created padded instrumental: {padded_instrumental}")
|
||||
|
||||
actual_instrumental = padded_instrumental
|
||||
else:
|
||||
self.logger.info(f"Using already-padded instrumental: {instrumental_audio}")
|
||||
|
||||
# This operation is primarily I/O bound (remuxing), so hardware acceleration doesn't provide significant benefit
|
||||
# Keep the existing approach but use the new execute method
|
||||
ffmpeg_command = (
|
||||
f'{self.ffmpeg_base_command} -an -i "{with_vocals_file}" '
|
||||
f'-vn -i "{instrumental_audio}" -c:v copy -c:a pcm_s16le "{output_file}"'
|
||||
f'-vn -i "{actual_instrumental}" -c:v copy -c:a pcm_s16le "{output_file}"'
|
||||
)
|
||||
self.execute_command(ffmpeg_command, "Remuxing video with instrumental audio")
|
||||
|
||||
|
||||
@@ -259,6 +259,101 @@ class KaraokePrep:
|
||||
self.artist = metadata_result["artist"]
|
||||
self.title = metadata_result["title"]
|
||||
|
||||
def _scan_directory_for_instrumentals(self, track_output_dir, artist_title):
|
||||
"""
|
||||
Scan the directory for existing instrumental files and build a separated_audio structure.
|
||||
|
||||
This is used when transcription was skipped (existing files found) but we need to
|
||||
pad instrumentals due to countdown padding.
|
||||
|
||||
Args:
|
||||
track_output_dir: The track output directory to scan
|
||||
artist_title: The "{artist} - {title}" string for matching files
|
||||
|
||||
Returns:
|
||||
Dictionary with separated_audio structure containing found instrumental paths
|
||||
"""
|
||||
self.logger.info(f"Scanning directory for existing instrumentals: {track_output_dir}")
|
||||
|
||||
separated_audio = {
|
||||
"clean_instrumental": {},
|
||||
"backing_vocals": {},
|
||||
"other_stems": {},
|
||||
"combined_instrumentals": {},
|
||||
}
|
||||
|
||||
# Search patterns for instrumental files
|
||||
# Files are named like: "{artist} - {title} (Instrumental {model}).flac"
|
||||
# Or with backing vocals: "{artist} - {title} (Instrumental +BV {model}).flac"
|
||||
|
||||
# Look for files in the track output directory
|
||||
search_dir = track_output_dir
|
||||
|
||||
# Find all instrumental files (not padded ones - we want the originals)
|
||||
instrumental_pattern = os.path.join(search_dir, f"{artist_title} (Instrumental*.flac")
|
||||
instrumental_files = glob.glob(instrumental_pattern)
|
||||
|
||||
# Also check for wav files
|
||||
instrumental_pattern_wav = os.path.join(search_dir, f"{artist_title} (Instrumental*.wav")
|
||||
instrumental_files.extend(glob.glob(instrumental_pattern_wav))
|
||||
|
||||
self.logger.debug(f"Found {len(instrumental_files)} instrumental files")
|
||||
|
||||
for filepath in instrumental_files:
|
||||
filename = os.path.basename(filepath)
|
||||
|
||||
# Skip already padded files
|
||||
if "(Padded)" in filename:
|
||||
self.logger.debug(f"Skipping already padded file: {filename}")
|
||||
continue
|
||||
|
||||
# Determine if it's a combined instrumental (+BV) or clean instrumental
|
||||
if "+BV" in filename or "+bv" in filename.lower():
|
||||
# Combined instrumental with backing vocals
|
||||
# Extract model name from filename
|
||||
# Pattern: "(Instrumental +BV {model}).flac"
|
||||
model_match = re.search(r'\(Instrumental \+BV ([^)]+)\)', filename)
|
||||
if model_match:
|
||||
model_name = model_match.group(1).strip()
|
||||
separated_audio["combined_instrumentals"][model_name] = filepath
|
||||
self.logger.info(f"Found combined instrumental: {filename}")
|
||||
else:
|
||||
# Clean instrumental (no backing vocals)
|
||||
# Pattern: "(Instrumental {model}).flac"
|
||||
model_match = re.search(r'\(Instrumental ([^)]+)\)', filename)
|
||||
if model_match:
|
||||
# Use as clean instrumental if we don't have one yet
|
||||
if not separated_audio["clean_instrumental"].get("instrumental"):
|
||||
separated_audio["clean_instrumental"]["instrumental"] = filepath
|
||||
self.logger.info(f"Found clean instrumental: {filename}")
|
||||
else:
|
||||
# Additional clean instrumentals go to combined_instrumentals for padding
|
||||
model_name = model_match.group(1).strip()
|
||||
separated_audio["combined_instrumentals"][model_name] = filepath
|
||||
self.logger.info(f"Found additional instrumental: {filename}")
|
||||
|
||||
# Also look for backing vocals files
|
||||
backing_vocals_pattern = os.path.join(search_dir, f"{artist_title} (Backing Vocals*.flac")
|
||||
backing_vocals_files = glob.glob(backing_vocals_pattern)
|
||||
backing_vocals_pattern_wav = os.path.join(search_dir, f"{artist_title} (Backing Vocals*.wav")
|
||||
backing_vocals_files.extend(glob.glob(backing_vocals_pattern_wav))
|
||||
|
||||
for filepath in backing_vocals_files:
|
||||
filename = os.path.basename(filepath)
|
||||
model_match = re.search(r'\(Backing Vocals ([^)]+)\)', filename)
|
||||
if model_match:
|
||||
model_name = model_match.group(1).strip()
|
||||
if model_name not in separated_audio["backing_vocals"]:
|
||||
separated_audio["backing_vocals"][model_name] = {"backing_vocals": filepath}
|
||||
self.logger.info(f"Found backing vocals: {filename}")
|
||||
|
||||
# Log summary
|
||||
clean_count = 1 if separated_audio["clean_instrumental"].get("instrumental") else 0
|
||||
combined_count = len(separated_audio["combined_instrumentals"])
|
||||
self.logger.info(f"Directory scan complete: {clean_count} clean instrumental, {combined_count} combined instrumentals")
|
||||
|
||||
return separated_audio
|
||||
|
||||
async def prep_single_track(self):
|
||||
# Add signal handler at the start
|
||||
loop = asyncio.get_running_loop()
|
||||
@@ -767,6 +862,18 @@ class KaraokePrep:
|
||||
f"Applying {padding_seconds}s padding to all instrumental files to sync with vocal countdown"
|
||||
)
|
||||
|
||||
# If separated_audio is empty (e.g., transcription was skipped but existing files have countdown),
|
||||
# scan the directory for existing instrumental files
|
||||
has_instrumentals = (
|
||||
processed_track["separated_audio"].get("clean_instrumental", {}).get("instrumental") or
|
||||
processed_track["separated_audio"].get("combined_instrumentals")
|
||||
)
|
||||
if not has_instrumentals:
|
||||
self.logger.info("No instrumentals in separated_audio, scanning directory for existing files...")
|
||||
processed_track["separated_audio"] = self._scan_directory_for_instrumentals(
|
||||
track_output_dir, artist_title
|
||||
)
|
||||
|
||||
# Apply padding using AudioProcessor
|
||||
padded_separation_result = self.audio_processor.apply_countdown_padding_to_instrumentals(
|
||||
separation_result=processed_track["separated_audio"],
|
||||
|
||||
@@ -11,6 +11,9 @@ from .utils import sanitize_filename
|
||||
|
||||
# Placeholder class or functions for lyrics processing
|
||||
class LyricsProcessor:
|
||||
# Standard countdown padding duration used by LyricsTranscriber
|
||||
COUNTDOWN_PADDING_SECONDS = 3.0
|
||||
|
||||
def __init__(
|
||||
self, logger, style_params_json, lyrics_file, skip_transcription, skip_transcription_review, render_video, subtitle_offset_ms
|
||||
):
|
||||
@@ -22,6 +25,60 @@ class LyricsProcessor:
|
||||
self.render_video = render_video
|
||||
self.subtitle_offset_ms = subtitle_offset_ms
|
||||
|
||||
def _detect_countdown_padding_from_lrc(self, lrc_filepath):
|
||||
"""
|
||||
Detect if countdown padding was applied by checking the first lyric timestamp in the LRC file.
|
||||
|
||||
LRC format timestamps look like: [mm:ss.xx] or [mm:ss.xxx]
|
||||
If the first lyric timestamp is >= 3.0 seconds, countdown padding was likely applied.
|
||||
|
||||
Args:
|
||||
lrc_filepath: Path to the LRC file
|
||||
|
||||
Returns:
|
||||
Tuple of (countdown_padding_added: bool, countdown_padding_seconds: float)
|
||||
"""
|
||||
try:
|
||||
with open(lrc_filepath, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Find all timestamp patterns in the LRC file
|
||||
# LRC timestamps: [mm:ss.xx] or [mm:ss.xxx]
|
||||
timestamp_pattern = r'\[(\d{1,2}):(\d{2})\.(\d{2,3})\]'
|
||||
matches = re.findall(timestamp_pattern, content)
|
||||
|
||||
if not matches:
|
||||
self.logger.debug("No timestamps found in LRC file")
|
||||
return (False, 0.0)
|
||||
|
||||
# Find the first non-metadata timestamp (metadata like [ar:Artist] doesn't have decimal)
|
||||
# We already filtered for decimal timestamps in our pattern
|
||||
first_timestamp = matches[0]
|
||||
minutes = int(first_timestamp[0])
|
||||
seconds = int(first_timestamp[1])
|
||||
# Handle both .xx and .xxx formats
|
||||
centiseconds = first_timestamp[2]
|
||||
if len(centiseconds) == 2:
|
||||
milliseconds = int(centiseconds) * 10
|
||||
else:
|
||||
milliseconds = int(centiseconds)
|
||||
|
||||
first_lyric_time = minutes * 60 + seconds + milliseconds / 1000.0
|
||||
|
||||
self.logger.debug(f"First lyric timestamp in LRC: {first_lyric_time:.3f}s")
|
||||
|
||||
# If first lyric is at or after 3 seconds, countdown padding was applied
|
||||
# Use a small buffer (2.5s) to account for songs that naturally start a bit late
|
||||
if first_lyric_time >= 2.5:
|
||||
self.logger.info(f"Detected countdown padding from LRC: first lyric at {first_lyric_time:.2f}s")
|
||||
return (True, self.COUNTDOWN_PADDING_SECONDS)
|
||||
|
||||
return (False, 0.0)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.warning(f"Failed to detect countdown padding from LRC file: {e}")
|
||||
return (False, 0.0)
|
||||
|
||||
def find_best_split_point(self, line):
|
||||
"""
|
||||
Find the best split point in a line based on the specified criteria.
|
||||
@@ -138,23 +195,43 @@ class LyricsProcessor:
|
||||
lyrics_video_path = os.path.join(lyrics_dir, f"{sanitized_artist} - {sanitized_title} (With Vocals).mkv")
|
||||
lyrics_lrc_path = os.path.join(lyrics_dir, f"{sanitized_artist} - {sanitized_title} (Karaoke).lrc")
|
||||
|
||||
# If files exist in parent directory, return early
|
||||
# If files exist in parent directory, return early (but detect countdown padding first)
|
||||
if os.path.exists(parent_video_path) and os.path.exists(parent_lrc_path):
|
||||
self.logger.info(f"Found existing video and LRC files in parent directory, skipping transcription")
|
||||
self.logger.info("Found existing video and LRC files in parent directory, skipping transcription")
|
||||
|
||||
# Detect countdown padding from existing LRC file
|
||||
countdown_padding_added, countdown_padding_seconds = self._detect_countdown_padding_from_lrc(parent_lrc_path)
|
||||
|
||||
if countdown_padding_added:
|
||||
self.logger.info(f"Existing files have countdown padding: {countdown_padding_seconds}s")
|
||||
|
||||
return {
|
||||
"lrc_filepath": parent_lrc_path,
|
||||
"ass_filepath": parent_video_path,
|
||||
"countdown_padding_added": countdown_padding_added,
|
||||
"countdown_padding_seconds": countdown_padding_seconds,
|
||||
"padded_audio_filepath": None, # Original padded audio may not exist
|
||||
}
|
||||
|
||||
# If files exist in lyrics directory, copy to parent and return
|
||||
# If files exist in lyrics directory, copy to parent and return (but detect countdown padding first)
|
||||
if os.path.exists(lyrics_video_path) and os.path.exists(lyrics_lrc_path):
|
||||
self.logger.info(f"Found existing video and LRC files in lyrics directory, copying to parent")
|
||||
self.logger.info("Found existing video and LRC files in lyrics directory, copying to parent")
|
||||
os.makedirs(track_output_dir, exist_ok=True)
|
||||
shutil.copy2(lyrics_video_path, parent_video_path)
|
||||
shutil.copy2(lyrics_lrc_path, parent_lrc_path)
|
||||
|
||||
# Detect countdown padding from existing LRC file
|
||||
countdown_padding_added, countdown_padding_seconds = self._detect_countdown_padding_from_lrc(parent_lrc_path)
|
||||
|
||||
if countdown_padding_added:
|
||||
self.logger.info(f"Existing files have countdown padding: {countdown_padding_seconds}s")
|
||||
|
||||
return {
|
||||
"lrc_filepath": parent_lrc_path,
|
||||
"ass_filepath": parent_video_path,
|
||||
"countdown_padding_added": countdown_padding_added,
|
||||
"countdown_padding_seconds": countdown_padding_seconds,
|
||||
"padded_audio_filepath": None, # Original padded audio may not exist
|
||||
}
|
||||
|
||||
# Create lyrics directory if it doesn't exist
|
||||
|
||||
@@ -820,6 +820,12 @@ async def async_main():
|
||||
|
||||
logger.info(f"Selected instrumental file: {selected_instrumental_file}")
|
||||
|
||||
# Get countdown padding info from track (if vocals were padded, instrumental must match)
|
||||
countdown_padding_seconds = None
|
||||
if track.get("countdown_padding_added", False):
|
||||
countdown_padding_seconds = track.get("countdown_padding_seconds", 3.0)
|
||||
logger.info(f"Countdown padding detected: {countdown_padding_seconds}s (will be applied to instrumental if needed)")
|
||||
|
||||
# Load CDG styles if CDG generation is enabled
|
||||
cdg_styles = None
|
||||
if args.enable_cdg:
|
||||
@@ -864,6 +870,7 @@ async def async_main():
|
||||
keep_brand_code=getattr(args, 'keep_brand_code', False),
|
||||
non_interactive=args.yes,
|
||||
selected_instrumental_file=selected_instrumental_file,
|
||||
countdown_padding_seconds=countdown_padding_seconds,
|
||||
)
|
||||
|
||||
try:
|
||||
|
||||
+112
-29
@@ -36,6 +36,8 @@ from typing import Any, Dict, Optional
|
||||
import requests
|
||||
|
||||
from .cli_args import create_parser, process_style_overrides, is_url, is_file
|
||||
# Use flacfetch's shared display functions for consistent formatting
|
||||
from flacfetch import print_releases, Release
|
||||
|
||||
|
||||
class JobStatus(str, Enum):
|
||||
@@ -1329,6 +1331,54 @@ class JobMonitor:
|
||||
"""Get user-friendly description for a status."""
|
||||
return self.STATUS_DESCRIPTIONS.get(status, status)
|
||||
|
||||
def _show_download_progress(self, job_data: Dict[str, Any]) -> None:
|
||||
"""Show detailed download progress during audio download."""
|
||||
try:
|
||||
# Get provider from job state_data
|
||||
state_data = job_data.get('state_data', {})
|
||||
provider = state_data.get('selected_audio_provider', 'unknown')
|
||||
|
||||
# For non-torrent providers (YouTube), just show simple message
|
||||
if provider.lower() == 'youtube':
|
||||
self.logger.info(f" [Downloading from YouTube...]")
|
||||
return
|
||||
|
||||
# Query health endpoint for transmission status (torrent providers)
|
||||
health_url = f"{self.config.service_url}/api/health/detailed"
|
||||
response = requests.get(health_url, timeout=5)
|
||||
|
||||
if response.status_code == 200:
|
||||
health = response.json()
|
||||
transmission = health.get('dependencies', {}).get('transmission', {})
|
||||
|
||||
if transmission.get('available'):
|
||||
torrents = transmission.get('torrents', [])
|
||||
if torrents:
|
||||
# Show info about active torrents
|
||||
for t in torrents:
|
||||
progress = t.get('progress', 0)
|
||||
peers = t.get('peers', 0)
|
||||
speed = t.get('download_speed', 0)
|
||||
stalled = t.get('stalled', False)
|
||||
|
||||
if stalled:
|
||||
self.logger.info(f" [Downloading from {provider}] {progress:.1f}% - STALLED (no peers)")
|
||||
elif progress < 100:
|
||||
self.logger.info(f" [Downloading from {provider}] {progress:.1f}% @ {speed:.1f} KB/s ({peers} peers)")
|
||||
else:
|
||||
self.logger.info(f" [Downloading from {provider}] Complete, processing...")
|
||||
else:
|
||||
# No torrents - might be starting or YouTube download
|
||||
self.logger.info(f" [Downloading from {provider}] Starting download...")
|
||||
else:
|
||||
self.logger.info(f" [Downloading from {provider}] Transmission not available - download may fail")
|
||||
else:
|
||||
self.logger.info(f" [Downloading from {provider}]...")
|
||||
|
||||
except Exception as e:
|
||||
# Fall back to simple message
|
||||
self.logger.info(f" [Downloading audio...]")
|
||||
|
||||
def open_browser(self, url: str) -> None:
|
||||
"""Open URL in the default browser."""
|
||||
system = platform.system()
|
||||
@@ -1517,6 +1567,47 @@ class JobMonitor:
|
||||
except Exception as e:
|
||||
self.logger.error(f"Error submitting selection: {e}")
|
||||
|
||||
def _convert_api_result_to_release_dict(self, result: dict) -> dict:
|
||||
"""
|
||||
Convert API search result to a dict compatible with flacfetch's Release.from_dict().
|
||||
|
||||
This enables using flacfetch's shared display functions for consistent,
|
||||
rich formatting between local and remote CLIs.
|
||||
"""
|
||||
# Build quality dict from API response
|
||||
quality_data = result.get('quality_data') or {
|
||||
"format": "OTHER",
|
||||
"media": "OTHER",
|
||||
}
|
||||
|
||||
return {
|
||||
"title": result.get('title', ''),
|
||||
"artist": result.get('artist', ''),
|
||||
"source_name": result.get('provider', 'Unknown'),
|
||||
"download_url": result.get('url'),
|
||||
"info_hash": result.get('source_id'),
|
||||
"size_bytes": result.get('size_bytes'),
|
||||
"year": result.get('year'),
|
||||
"edition_info": result.get('edition_info'),
|
||||
"label": result.get('label'),
|
||||
"release_type": result.get('release_type'),
|
||||
"seeders": result.get('seeders'),
|
||||
"channel": result.get('channel'),
|
||||
"view_count": result.get('view_count'),
|
||||
"duration_seconds": result.get('duration'),
|
||||
"target_file": result.get('target_file'),
|
||||
"target_file_size": result.get('target_file_size'),
|
||||
"track_pattern": result.get('track_pattern'),
|
||||
"match_score": result.get('match_score', 0.0),
|
||||
"quality": quality_data,
|
||||
# Pre-computed fields
|
||||
"formatted_size": result.get('formatted_size'),
|
||||
"formatted_duration": result.get('formatted_duration'),
|
||||
"formatted_views": result.get('formatted_views'),
|
||||
"is_lossless": result.get('is_lossless', False),
|
||||
"quality_str": result.get('quality_str') or result.get('quality', ''),
|
||||
}
|
||||
|
||||
def handle_audio_selection(self, job_id: str) -> None:
|
||||
"""Handle audio source selection interaction (Batch 5)."""
|
||||
self.logger.info("=" * 60)
|
||||
@@ -1527,6 +1618,7 @@ class JobMonitor:
|
||||
# Get search results
|
||||
results_data = self.client.get_audio_search_results(job_id)
|
||||
results = results_data.get('results', [])
|
||||
artist = results_data.get('artist', 'Unknown')
|
||||
|
||||
if not results:
|
||||
self.logger.error("No search results available")
|
||||
@@ -1537,42 +1629,27 @@ class JobMonitor:
|
||||
self.logger.info("Non-interactive mode: Auto-selecting first result")
|
||||
selection_index = 0
|
||||
else:
|
||||
self.logger.info("")
|
||||
self.logger.info("Choose which audio source to download:")
|
||||
self.logger.info("")
|
||||
# Convert API results to Release-compatible dicts for flacfetch display
|
||||
# This gives us the same rich, colorized output as the local CLI
|
||||
release_dicts = [self._convert_api_result_to_release_dict(r) for r in results]
|
||||
|
||||
for result in results:
|
||||
index = result.get('index', 0)
|
||||
provider = result.get('provider', 'Unknown')
|
||||
artist = result.get('artist', 'Unknown')
|
||||
title = result.get('title', 'Unknown')
|
||||
quality = result.get('quality', '')
|
||||
duration = result.get('duration')
|
||||
|
||||
# Format duration if available
|
||||
duration_str = ""
|
||||
if duration:
|
||||
minutes = duration // 60
|
||||
seconds = duration % 60
|
||||
duration_str = f" [{minutes}:{seconds:02d}]"
|
||||
|
||||
quality_str = f" ({quality})" if quality else ""
|
||||
|
||||
self.logger.info(f" {index + 1}) [{provider}] {artist} - {title}{quality_str}{duration_str}")
|
||||
|
||||
self.logger.info("")
|
||||
# Use flacfetch's shared display function
|
||||
print_releases(release_dicts, target_artist=artist, use_colors=True)
|
||||
|
||||
selection_index = -1
|
||||
while selection_index < 0:
|
||||
try:
|
||||
choice = input(f"Enter your choice (1-{len(results)}): ").strip()
|
||||
choice = input(f"\nSelect a release (1-{len(results)}, 0 to cancel): ").strip()
|
||||
if choice == "0":
|
||||
self.logger.info("Selection cancelled by user")
|
||||
raise KeyboardInterrupt
|
||||
choice_num = int(choice)
|
||||
if 1 <= choice_num <= len(results):
|
||||
selection_index = choice_num - 1
|
||||
else:
|
||||
self.logger.error(f"Please enter a number between 1 and {len(results)}")
|
||||
print(f"Please enter a number between 0 and {len(results)}")
|
||||
except ValueError:
|
||||
self.logger.error("Please enter a valid number")
|
||||
print("Please enter a valid number")
|
||||
except KeyboardInterrupt:
|
||||
print()
|
||||
raise
|
||||
@@ -2038,9 +2115,15 @@ class JobMonitor:
|
||||
self._polls_without_updates = 0
|
||||
else:
|
||||
self._polls_without_updates += 1
|
||||
if self._polls_without_updates >= self._heartbeat_interval:
|
||||
description = self._get_status_description(status)
|
||||
self.logger.info(f" [Still processing: {description}]")
|
||||
# More frequent updates during audio download (every poll)
|
||||
heartbeat_threshold = 1 if status == 'downloading_audio' else self._heartbeat_interval
|
||||
if self._polls_without_updates >= heartbeat_threshold:
|
||||
if status == 'downloading_audio':
|
||||
# Show detailed download progress including transmission status
|
||||
self._show_download_progress(job_data)
|
||||
else:
|
||||
description = self._get_status_description(status)
|
||||
self.logger.info(f" [Still processing: {description}]")
|
||||
self._polls_without_updates = 0
|
||||
|
||||
# Handle human interaction points
|
||||
|
||||
Generated
+403
-407
File diff suppressed because it is too large.
Load diff
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.71.69"
|
||||
version = "0.73.3"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -435,6 +435,266 @@ class TestExistingInstrumentalPadding:
|
||||
assert '3.0' in call_args or 3.0 in call_args
|
||||
|
||||
|
||||
class TestDetectCountdownPaddingFromLRC:
|
||||
"""Tests for detecting countdown padding from existing LRC files."""
|
||||
|
||||
def test_detect_padding_from_lrc_with_padding(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test that countdown padding is detected when first lyric is after 3 seconds."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
# Create an LRC file with first lyric at 3 seconds (padding was applied)
|
||||
lrc_content = """[ti:Test Song]
|
||||
[ar:Test Artist]
|
||||
[00:03.50]First line of lyrics
|
||||
[00:06.20]Second line of lyrics
|
||||
[00:10.00]Third line of lyrics
|
||||
"""
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
assert padding_added is True
|
||||
assert padding_seconds == 3.0
|
||||
|
||||
def test_detect_padding_from_lrc_without_padding(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test that no padding is detected when first lyric is before 3 seconds."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
# Create an LRC file with first lyric at 0.5 seconds (no padding)
|
||||
lrc_content = """[ti:Test Song]
|
||||
[ar:Test Artist]
|
||||
[00:00.50]First line of lyrics
|
||||
[00:03.20]Second line of lyrics
|
||||
[00:06.00]Third line of lyrics
|
||||
"""
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
assert padding_added is False
|
||||
assert padding_seconds == 0.0
|
||||
|
||||
def test_detect_padding_from_lrc_edge_case(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test detection at the 2.5 second boundary."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
# Create an LRC file with first lyric at exactly 2.5 seconds (boundary)
|
||||
lrc_content = """[00:02.50]First line at boundary
|
||||
[00:05.00]Second line
|
||||
"""
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
# 2.5 is the boundary - should be detected as padding
|
||||
assert padding_added is True
|
||||
assert padding_seconds == 3.0
|
||||
|
||||
def test_detect_padding_from_lrc_just_below_boundary(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test detection just below the 2.5 second boundary."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
# Create an LRC file with first lyric at 2.4 seconds (just below boundary)
|
||||
lrc_content = """[00:02.40]First line just below boundary
|
||||
[00:05.00]Second line
|
||||
"""
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
# 2.4 is below boundary - should NOT be detected as padding
|
||||
assert padding_added is False
|
||||
assert padding_seconds == 0.0
|
||||
|
||||
def test_detect_padding_from_lrc_empty_file(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test handling of empty LRC file."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write('')
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
assert padding_added is False
|
||||
assert padding_seconds == 0.0
|
||||
|
||||
def test_detect_padding_from_lrc_no_timestamps(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test handling of LRC file with no valid timestamps."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
lrc_content = """[ti:Test Song]
|
||||
[ar:Test Artist]
|
||||
Just text without timestamps
|
||||
More text
|
||||
"""
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
assert padding_added is False
|
||||
assert padding_seconds == 0.0
|
||||
|
||||
def test_detect_padding_from_lrc_missing_file(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test handling of missing LRC file."""
|
||||
lrc_file = os.path.join(temp_dir, 'nonexistent.lrc')
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
assert padding_added is False
|
||||
assert padding_seconds == 0.0
|
||||
|
||||
def test_detect_padding_from_lrc_three_digit_milliseconds(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test parsing LRC files with three-digit milliseconds."""
|
||||
lrc_file = os.path.join(temp_dir, 'test.lrc')
|
||||
|
||||
# Some LRC files use [mm:ss.xxx] format
|
||||
lrc_content = """[00:03.500]First line at 3.5 seconds
|
||||
[00:06.200]Second line
|
||||
"""
|
||||
with open(lrc_file, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
padding_added, padding_seconds = basic_karaoke_gen.lyrics_processor._detect_countdown_padding_from_lrc(lrc_file)
|
||||
|
||||
assert padding_added is True
|
||||
assert padding_seconds == 3.0
|
||||
|
||||
|
||||
class TestExistingFilesWithPaddingDetection:
|
||||
"""Tests for transcribe_lyrics returning padding info from existing files."""
|
||||
|
||||
def test_transcribe_lyrics_existing_files_returns_padding_info(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test that transcribe_lyrics returns padding info when using existing files."""
|
||||
track_output_dir = temp_dir
|
||||
artist = "Test Artist"
|
||||
title = "Test Title"
|
||||
|
||||
# Create existing files
|
||||
video_path = os.path.join(track_output_dir, f"{artist} - {title} (With Vocals).mkv")
|
||||
lrc_path = os.path.join(track_output_dir, f"{artist} - {title} (Karaoke).lrc")
|
||||
|
||||
# Create the video file
|
||||
with open(video_path, 'w') as f:
|
||||
f.write('mock video')
|
||||
|
||||
# Create LRC file with countdown padding (first lyric at 3.5s)
|
||||
lrc_content = """[00:03.50]First line with padding
|
||||
[00:06.00]Second line
|
||||
"""
|
||||
with open(lrc_path, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
# Call transcribe_lyrics - it should detect existing files and return padding info
|
||||
result = basic_karaoke_gen.lyrics_processor.transcribe_lyrics(
|
||||
input_audio_wav=None, # Not needed when existing files found
|
||||
artist=artist,
|
||||
title=title,
|
||||
track_output_dir=track_output_dir,
|
||||
)
|
||||
|
||||
# Should return padding info from LRC detection
|
||||
assert result['countdown_padding_added'] is True
|
||||
assert result['countdown_padding_seconds'] == 3.0
|
||||
assert result['padded_audio_filepath'] is None # Original padded audio not available
|
||||
|
||||
def test_transcribe_lyrics_existing_files_no_padding(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test existing files case when there's no countdown padding."""
|
||||
track_output_dir = temp_dir
|
||||
artist = "Test Artist"
|
||||
title = "Test Title"
|
||||
|
||||
# Create existing files
|
||||
video_path = os.path.join(track_output_dir, f"{artist} - {title} (With Vocals).mkv")
|
||||
lrc_path = os.path.join(track_output_dir, f"{artist} - {title} (Karaoke).lrc")
|
||||
|
||||
# Create the video file
|
||||
with open(video_path, 'w') as f:
|
||||
f.write('mock video')
|
||||
|
||||
# Create LRC file WITHOUT countdown padding (first lyric at 0.5s)
|
||||
lrc_content = """[00:00.50]First line starts immediately
|
||||
[00:03.00]Second line
|
||||
"""
|
||||
with open(lrc_path, 'w') as f:
|
||||
f.write(lrc_content)
|
||||
|
||||
result = basic_karaoke_gen.lyrics_processor.transcribe_lyrics(
|
||||
input_audio_wav=None,
|
||||
artist=artist,
|
||||
title=title,
|
||||
track_output_dir=track_output_dir,
|
||||
)
|
||||
|
||||
# Should return no padding info
|
||||
assert result['countdown_padding_added'] is False
|
||||
assert result['countdown_padding_seconds'] == 0.0
|
||||
|
||||
|
||||
class TestScanDirectoryForInstrumentals:
|
||||
"""Tests for scanning directory for existing instrumentals."""
|
||||
|
||||
def test_scan_directory_finds_clean_instrumental(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test that _scan_directory_for_instrumentals finds clean instrumentals."""
|
||||
artist_title = "Test Artist - Test Song"
|
||||
|
||||
# Create a clean instrumental file
|
||||
instrumental_file = os.path.join(temp_dir, f"{artist_title} (Instrumental Clean).flac")
|
||||
with open(instrumental_file, 'w') as f:
|
||||
f.write('mock instrumental')
|
||||
|
||||
result = basic_karaoke_gen._scan_directory_for_instrumentals(temp_dir, artist_title)
|
||||
|
||||
assert result['clean_instrumental']['instrumental'] == instrumental_file
|
||||
|
||||
def test_scan_directory_finds_combined_instrumentals(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test that _scan_directory_for_instrumentals finds combined instrumentals."""
|
||||
artist_title = "Test Artist - Test Song"
|
||||
|
||||
# Create combined instrumental files
|
||||
bv_file = os.path.join(temp_dir, f"{artist_title} (Instrumental +BV HTDemucs).flac")
|
||||
with open(bv_file, 'w') as f:
|
||||
f.write('mock instrumental with backing')
|
||||
|
||||
result = basic_karaoke_gen._scan_directory_for_instrumentals(temp_dir, artist_title)
|
||||
|
||||
assert 'HTDemucs' in result['combined_instrumentals']
|
||||
assert result['combined_instrumentals']['HTDemucs'] == bv_file
|
||||
|
||||
def test_scan_directory_skips_padded_files(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test that _scan_directory_for_instrumentals skips already-padded files."""
|
||||
artist_title = "Test Artist - Test Song"
|
||||
|
||||
# Create both original and padded files
|
||||
original_file = os.path.join(temp_dir, f"{artist_title} (Instrumental Clean).flac")
|
||||
padded_file = os.path.join(temp_dir, f"{artist_title} (Instrumental Clean) (Padded).flac")
|
||||
|
||||
with open(original_file, 'w') as f:
|
||||
f.write('mock original')
|
||||
with open(padded_file, 'w') as f:
|
||||
f.write('mock padded')
|
||||
|
||||
result = basic_karaoke_gen._scan_directory_for_instrumentals(temp_dir, artist_title)
|
||||
|
||||
# Should return original, not padded
|
||||
assert result['clean_instrumental']['instrumental'] == original_file
|
||||
assert 'Padded' not in result['clean_instrumental']['instrumental']
|
||||
|
||||
def test_scan_directory_empty_directory(self, basic_karaoke_gen, temp_dir):
|
||||
"""Test _scan_directory_for_instrumentals on empty directory."""
|
||||
artist_title = "Test Artist - Test Song"
|
||||
|
||||
result = basic_karaoke_gen._scan_directory_for_instrumentals(temp_dir, artist_title)
|
||||
|
||||
# Should return empty structure
|
||||
assert result['clean_instrumental'].get('instrumental') is None
|
||||
assert len(result['combined_instrumentals']) == 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
|
||||
@@ -2038,3 +2038,201 @@ class TestSignedUrlUploadFlow:
|
||||
assert hasattr(client, '_get_content_type')
|
||||
assert callable(client._upload_file_to_signed_url)
|
||||
assert callable(client._get_content_type)
|
||||
|
||||
|
||||
class TestJobMonitorDownloadProgress:
|
||||
"""Tests for JobMonitor download progress display."""
|
||||
|
||||
@pytest.fixture
|
||||
def config(self):
|
||||
return Config(
|
||||
service_url="https://api.example.com",
|
||||
review_ui_url="https://review.example.com",
|
||||
poll_interval=1,
|
||||
output_dir="/tmp/output"
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def logger(self):
|
||||
return logging.getLogger("test")
|
||||
|
||||
@pytest.fixture
|
||||
def monitor(self, config, logger):
|
||||
client = MagicMock()
|
||||
return JobMonitor(client, config, logger)
|
||||
|
||||
def test_show_download_progress_youtube(self, monitor, caplog):
|
||||
"""Test that YouTube downloads show simple message."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'YouTube'
|
||||
}
|
||||
}
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "Downloading from YouTube" in caplog.text
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.requests.get')
|
||||
def test_show_download_progress_torrent_with_active_torrents(self, mock_get, monitor, caplog):
|
||||
"""Test that active torrents show progress details."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'Redacted'
|
||||
}
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
'dependencies': {
|
||||
'transmission': {
|
||||
'available': True,
|
||||
'torrents': [{
|
||||
'progress': 45.5,
|
||||
'peers': 3,
|
||||
'download_speed': 1250.0,
|
||||
'stalled': False
|
||||
}]
|
||||
}
|
||||
}
|
||||
}
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "Downloading from Redacted" in caplog.text
|
||||
assert "45.5%" in caplog.text
|
||||
assert "1250.0 KB/s" in caplog.text
|
||||
assert "3 peers" in caplog.text
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.requests.get')
|
||||
def test_show_download_progress_torrent_stalled(self, mock_get, monitor, caplog):
|
||||
"""Test that stalled torrents show warning."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'OPS'
|
||||
}
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
'dependencies': {
|
||||
'transmission': {
|
||||
'available': True,
|
||||
'torrents': [{
|
||||
'progress': 0.0,
|
||||
'peers': 0,
|
||||
'download_speed': 0,
|
||||
'stalled': True
|
||||
}]
|
||||
}
|
||||
}
|
||||
}
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "STALLED" in caplog.text
|
||||
assert "no peers" in caplog.text
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.requests.get')
|
||||
def test_show_download_progress_no_torrents(self, mock_get, monitor, caplog):
|
||||
"""Test message when no active torrents."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'Redacted'
|
||||
}
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
'dependencies': {
|
||||
'transmission': {
|
||||
'available': True,
|
||||
'torrents': []
|
||||
}
|
||||
}
|
||||
}
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "Starting download" in caplog.text
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.requests.get')
|
||||
def test_show_download_progress_transmission_unavailable(self, mock_get, monitor, caplog):
|
||||
"""Test warning when Transmission not available."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'Redacted'
|
||||
}
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.json.return_value = {
|
||||
'dependencies': {
|
||||
'transmission': {
|
||||
'available': False
|
||||
}
|
||||
}
|
||||
}
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "not available" in caplog.text
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.requests.get')
|
||||
def test_show_download_progress_request_error(self, mock_get, monitor, caplog):
|
||||
"""Test fallback message on request error."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'Redacted'
|
||||
}
|
||||
}
|
||||
|
||||
mock_get.side_effect = Exception("Connection error")
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "Downloading audio" in caplog.text
|
||||
|
||||
@patch('karaoke_gen.utils.remote_cli.requests.get')
|
||||
def test_show_download_progress_http_error(self, mock_get, monitor, caplog):
|
||||
"""Test fallback on non-200 HTTP response."""
|
||||
job_data = {
|
||||
'state_data': {
|
||||
'selected_audio_provider': 'Redacted'
|
||||
}
|
||||
}
|
||||
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = 500
|
||||
mock_get.return_value = mock_response
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
assert "Downloading from Redacted" in caplog.text
|
||||
|
||||
def test_show_download_progress_missing_provider(self, monitor, caplog):
|
||||
"""Test handling of missing provider in state_data."""
|
||||
job_data = {
|
||||
'state_data': {}
|
||||
}
|
||||
|
||||
with caplog.at_level(logging.INFO):
|
||||
monitor._show_download_progress(job_data)
|
||||
|
||||
# Should use 'unknown' as default
|
||||
assert "unknown" in caplog.text.lower() or "Downloading" in caplog.text
|
||||
Reference in new issue
Block a user