Compare commits

..
Author SHA1 Message Date
Andrew Beveridge f99d6c1fb4 style: remove extraneous f-string prefixes
Address CodeRabbit nitpick comments - remove f-string prefixes from
strings without placeholders.
2025-12-19 18:42:03 -05:00
Andrew Beveridge f8a45bfbd2 fix: countdown padding synchronization for instrumental audio
When LyricsTranscriber adds 3-second countdown padding to vocals,
the instrumental audio must be padded to match. This was broken when
transcription was skipped due to existing files.

Changes:
- Add _detect_countdown_padding_from_lrc() to detect padding from existing LRC
- Update early returns in transcribe_lyrics() to include padding info
- Add _scan_directory_for_instrumentals() to find existing instrumentals
- Fix counting bug in apply_countdown_padding_to_instrumentals
- Add countdown_padding_seconds safety net to KaraokeFinalise
- Store padding metadata in lyrics_worker job state_data
- Pass padding info to video_worker for instrumental sync
- Add comprehensive unit tests for new functionality
2025-12-19 18:35:37 -05:00
25 changed files with 891 additions and 4299 deletions

No files matched your search

+4 -9
View File
@@ -415,9 +415,8 @@ jobs:
run: |
echo "🔍 Checking if base image needs rebuild..."
# Hash both poetry.lock AND Dockerfile.base - rebuild if either changes
CURRENT_HASH=$(cat poetry.lock backend/Dockerfile.base | sha256sum | cut -d' ' -f1)
echo "Current combined hash (poetry.lock + Dockerfile.base): $CURRENT_HASH"
CURRENT_HASH=$(sha256sum poetry.lock | cut -d' ' -f1)
echo "Current poetry.lock hash: $CURRENT_HASH"
gcloud auth configure-docker us-central1-docker.pkg.dev --quiet
@@ -425,18 +424,14 @@ jobs:
EXISTING_HASH=""
if docker pull $BASE_IMAGE 2>/dev/null; then
EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "base.content.hash"}}' 2>/dev/null || echo "")
# Fallback to old label name for backwards compatibility
if [ -z "$EXISTING_HASH" ]; then
EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "poetry.lock.hash"}}' 2>/dev/null || echo "")
fi
EXISTING_HASH=$(docker inspect $BASE_IMAGE --format='{{index .Config.Labels "poetry.lock.hash"}}' 2>/dev/null || echo "")
echo "Existing base image hash: $EXISTING_HASH"
else
echo "⚠️ No existing base image found"
fi
if [ "$CURRENT_HASH" != "$EXISTING_HASH" ]; then
echo "🔄 Base image needs rebuild (poetry.lock or Dockerfile.base changed)"
echo "🔄 Base image needs rebuild (poetry.lock changed)"
echo "needs_rebuild=true" >> $GITHUB_OUTPUT
else
echo "✅ Base image is up to date"
+2 -6
View File
@@ -11,9 +11,7 @@
# gcloud builds submit --config=cloudbuild-base.yaml --project=nomadkaraoke
# Use pre-built base image with all dependencies
# Override with --build-arg BASE_IMAGE=karaoke-backend-base:local for local testing
ARG BASE_IMAGE=us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest
FROM ${BASE_IMAGE}
FROM us-central1-docker.pkg.dev/nomadkaraoke/karaoke-repo/karaoke-backend-base:latest
# Set working directory (should already exist from base)
WORKDIR /app
@@ -36,7 +34,5 @@ ENV PORT=8080
# Expose port
EXPOSE 8080
# Run uvicorn directly
# Note: Transmission daemon removed - torrent downloads now handled by
# dedicated flacfetch VM service (see docs/00-current-plan/TORRENT-SERVICE-PLAN.md)
# Run the application
CMD ["uvicorn", "backend.main:app", "--host", "0.0.0.0", "--port", "8080"]
+3 -6
View File
@@ -7,15 +7,12 @@
FROM python:3.11-slim
# Install system dependencies required by karaoke-gen
# Note: Transmission daemon removed - torrent downloads now handled by
# dedicated flacfetch VM service (see docs/00-current-plan/TORRENT-SERVICE-PLAN.md)
RUN apt-get update && apt-get install -y \
ffmpeg \
libsndfile1 \
libsox-dev \
sox \
build-essential \
curl \
&& rm -rf /var/lib/apt/lists/*
# Set working directory
@@ -37,9 +34,9 @@ RUN python -m spacy download en_core_web_sm
# Label with build info
ARG BUILD_DATE
ARG BUILD_ID
ARG BASE_CONTENT_HASH
ARG POETRY_LOCK_HASH
LABEL org.opencontainers.image.created="${BUILD_DATE}"
LABEL org.opencontainers.image.revision="${BUILD_ID}"
LABEL org.opencontainers.image.title="karaoke-backend-base"
LABEL org.opencontainers.image.description="Base image with Python dependencies and system tools for karaoke-backend"
LABEL base.content.hash="${BASE_CONTENT_HASH}"
LABEL org.opencontainers.image.description="Base image with Python dependencies for karaoke-backend"
LABEL poetry.lock.hash="${POETRY_LOCK_HASH}"
+19 -76
View File
@@ -82,40 +82,15 @@ class AudioSearchRequest(BaseModel):
class AudioSearchResultResponse(BaseModel):
"""A single audio search result.
Contains all fields needed for rich display using flacfetch's
shared formatting functions.
"""
"""A single audio search result."""
index: int
title: str
artist: str
provider: str # Maps to source_name in Release
url: str # Maps to download_url in Release
duration: Optional[int] = None # Maps to duration_seconds in Release
quality: Optional[str] = None # Stringified quality
source_id: Optional[str] = None # Maps to info_hash in Release
seeders: Optional[int] = None
target_file: Optional[str] = None
# Additional fields for rich display (from Release.to_dict())
year: Optional[int] = None
label: Optional[str] = None
edition_info: Optional[str] = None
release_type: Optional[str] = None
channel: Optional[str] = None # For YouTube
view_count: Optional[int] = None # For YouTube
size_bytes: Optional[int] = None
target_file_size: Optional[int] = None
track_pattern: Optional[str] = None
match_score: Optional[float] = None
# Pre-computed display fields
formatted_size: Optional[str] = None
formatted_duration: Optional[str] = None
formatted_views: Optional[str] = None
is_lossless: Optional[bool] = None
quality_str: Optional[str] = None
# Full quality object for Release.from_dict()
quality_data: Optional[Dict[str, Any]] = None
provider: str
url: str
duration: Optional[int] = None
quality: Optional[str] = None
source_id: Optional[str] = None
class AudioSearchResponse(BaseModel):
@@ -443,50 +418,20 @@ async def search_audio(
message=f"Found {len(search_results)} audio sources. Waiting for selection."
)
# Convert to response format with full Release data for rich display
result_responses = []
for r in search_results:
# Get full serialized data from raw_result if available
raw_dict = {}
if r.raw_result:
try:
raw_dict = r.raw_result.to_dict()
except AttributeError:
pass # Not a Release object
result_responses.append(
AudioSearchResultResponse(
index=r.index,
title=r.title,
artist=r.artist,
provider=r.provider,
url=r.url,
duration=r.duration,
quality=r.quality,
source_id=r.source_id,
seeders=raw_dict.get('seeders') or r.seeders,
target_file=raw_dict.get('target_file') or r.target_file,
# Additional fields for rich display
year=raw_dict.get('year'),
label=raw_dict.get('label'),
edition_info=raw_dict.get('edition_info'),
release_type=raw_dict.get('release_type'),
channel=raw_dict.get('channel'),
view_count=raw_dict.get('view_count'),
size_bytes=raw_dict.get('size_bytes'),
target_file_size=raw_dict.get('target_file_size'),
track_pattern=raw_dict.get('track_pattern'),
match_score=raw_dict.get('match_score'),
# Pre-computed display fields
formatted_size=raw_dict.get('formatted_size'),
formatted_duration=raw_dict.get('formatted_duration'),
formatted_views=raw_dict.get('formatted_views'),
is_lossless=raw_dict.get('is_lossless'),
quality_str=raw_dict.get('quality_str'),
# Full quality object for Release.from_dict()
quality_data=raw_dict.get('quality'),
)
# Convert to response format
result_responses = [
AudioSearchResultResponse(
index=r.index,
title=r.title,
artist=r.artist,
provider=r.provider,
url=r.url,
duration=r.duration,
quality=r.quality,
source_id=r.source_id,
)
for r in search_results
]
return AudioSearchResponse(
status="awaiting_selection",
@@ -528,8 +473,6 @@ async def get_audio_search_results(job_id: str):
"status": "success",
"job_id": job_id,
"job_status": job.status,
"artist": job.audio_search_artist or job.artist,
"title": job.audio_search_title or job.title,
"results": search_results,
"results_count": len(search_results),
}
+1 -116
View File
@@ -1,64 +1,10 @@
"""
Health check routes.
"""
import os
import logging
from fastapi import APIRouter
from typing import Dict, Any, Optional
from backend.version import VERSION
from backend.services.flacfetch_client import get_flacfetch_client
from typing import Dict
router = APIRouter()
logger = logging.getLogger(__name__)
def check_transmission_status() -> Dict[str, Any]:
"""Check if Transmission daemon is available and responsive."""
try:
import transmission_rpc
host = os.environ.get("TRANSMISSION_HOST", "localhost")
port = int(os.environ.get("TRANSMISSION_PORT", "9091"))
client = transmission_rpc.Client(host=host, port=port, timeout=5)
stats = client.session_stats()
# Get detailed torrent info
torrents = client.get_torrents()
torrent_details = []
for t in torrents:
torrent_info = {
"name": t.name,
"progress": round(t.progress, 1),
"status": str(t.status),
"peers": t.peers_connected if hasattr(t, 'peers_connected') else 0,
"download_speed": round(t.rate_download / 1024, 1) if hasattr(t, 'rate_download') else 0, # KB/s
}
# Check if stalled (downloading but no progress and no peers)
if t.status.downloading and t.progress < 100 and torrent_info['peers'] == 0:
torrent_info['stalled'] = True
torrent_details.append(torrent_info)
return {
"available": True,
"host": host,
"port": port,
"download_dir": stats.download_dir if hasattr(stats, 'download_dir') else None,
"active_torrent_count": stats.active_torrent_count if hasattr(stats, 'active_torrent_count') else 0,
"torrents": torrent_details,
}
except ImportError as e:
return {
"available": False,
"error": f"transmission_rpc not installed: {e}",
}
except Exception as e:
return {
"available": False,
"host": os.environ.get("TRANSMISSION_HOST", "localhost"),
"port": int(os.environ.get("TRANSMISSION_PORT", "9091")),
"error": str(e),
}
@router.get("/health")
@@ -70,67 +16,6 @@ async def health_check() -> Dict[str, str]:
}
async def check_flacfetch_service_status() -> Dict[str, Any]:
"""Check if remote flacfetch service is available and healthy."""
client = get_flacfetch_client()
if not client:
return {
"configured": False,
"message": "Remote flacfetch service not configured (FLACFETCH_API_URL not set)",
}
try:
health = await client.health_check()
return {
"configured": True,
"available": True,
"status": health.get("status"),
"version": health.get("version"),
"transmission": health.get("transmission", {}),
"disk": health.get("disk", {}),
"providers": health.get("providers", {}),
}
except Exception as e:
return {
"configured": True,
"available": False,
"error": str(e),
}
@router.get("/health/detailed")
async def detailed_health_check() -> Dict[str, Any]:
"""
Detailed health check including dependencies.
Use this to debug issues with Transmission, flacfetch service, etc.
"""
transmission_status = check_transmission_status()
flacfetch_status = await check_flacfetch_service_status()
# Log for debugging
if not transmission_status.get("available"):
logger.warning(f"Local Transmission not available: {transmission_status.get('error')}")
else:
logger.info(f"Local Transmission available at {transmission_status.get('host')}:{transmission_status.get('port')}")
if flacfetch_status.get("configured") and not flacfetch_status.get("available"):
logger.warning(f"Remote flacfetch service not available: {flacfetch_status.get('error')}")
elif flacfetch_status.get("available"):
logger.info(f"Remote flacfetch service healthy: {flacfetch_status.get('status')}")
return {
"status": "healthy",
"service": "karaoke-gen-backend",
"version": VERSION,
"dependencies": {
"transmission_local": transmission_status,
"flacfetch_remote": flacfetch_status,
}
}
@router.get("/readiness")
async def readiness_check() -> Dict[str, str]:
"""Readiness check endpoint for Cloud Run."""
-6
View File
@@ -56,12 +56,6 @@ class Settings(BaseSettings):
# Storage paths
temp_dir: str = os.getenv("TEMP_DIR", "/tmp/karaoke-gen")
# Flacfetch remote service (for torrent downloads)
# When configured, audio search uses the remote flacfetch HTTP API instead of local flacfetch.
# This is required for torrent downloads since Cloud Run doesn't support BitTorrent.
flacfetch_api_url: Optional[str] = os.getenv("FLACFETCH_API_URL") # e.g., http://10.0.0.5:8080
flacfetch_api_key: Optional[str] = os.getenv("FLACFETCH_API_KEY")
# Default distribution settings (can be overridden per-request)
default_dropbox_path: Optional[str] = os.getenv("DEFAULT_DROPBOX_PATH")
default_gdrive_folder_id: Optional[str] = os.getenv("DEFAULT_GDRIVE_FOLDER_ID")
+194 -305
View File
@@ -1,67 +1,79 @@
"""
Audio search service for finding and downloading audio files.
This service integrates with karaoke_gen.audio_fetcher (which wraps flacfetch)
to search for audio from various sources (YouTube, music trackers, etc.)
and download the selected audio file.
This is a thin wrapper that adds backend-specific functionality:
- Caching raw results for API-based selection flow
- Firestore-compatible serialization
- Singleton pattern for service lifecycle
- Remote flacfetch service integration for torrent downloads
When FLACFETCH_API_URL is configured, this service uses the remote flacfetch
HTTP API for search and download operations. This is required for torrent
downloads since Cloud Run doesn't support BitTorrent peer connections.
This service integrates with flacfetch to search for audio from various sources
(YouTube, music trackers, etc.) and download the selected audio file.
"""
import asyncio
import logging
import os
from typing import List, Optional
# Import shared classes from karaoke_gen.audio_fetcher - single source of truth
# Note: Import directly from audio_fetcher module, not karaoke_gen.__init__
# to avoid pulling in heavier dependencies like lyrics_transcriber
from karaoke_gen.audio_fetcher import (
FlacFetcher,
AudioSearchResult,
AudioFetchResult,
AudioFetcherError,
NoResultsError,
DownloadError,
)
from .flacfetch_client import get_flacfetch_client, FlacfetchServiceError
import tempfile
from dataclasses import dataclass, asdict
from typing import List, Optional, Dict, Any
logger = logging.getLogger(__name__)
# Re-export exception under backend's naming convention for compatibility
AudioSearchError = AudioFetcherError
@dataclass
class AudioSearchResult:
"""
Represents a single search result for audio.
This is a serializable representation of a search result that can be
stored in Firestore and sent to clients.
"""
title: str
artist: str
provider: str
url: str
duration: Optional[int] = None # Duration in seconds
quality: Optional[str] = None # e.g., "FLAC", "320kbps", etc.
source_id: Optional[str] = None # Unique ID from the source
index: int = 0 # Index in the results list (for selection)
def to_dict(self) -> Dict[str, Any]:
"""Convert to dict for JSON serialization."""
return asdict(self)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "AudioSearchResult":
"""Create from dict."""
return cls(**data)
# Also alias AudioFetchResult as AudioDownloadResult for backwards compatibility
AudioDownloadResult = AudioFetchResult
@dataclass
class AudioDownloadResult:
"""Result of downloading an audio file."""
filepath: str
artist: str
title: str
provider: str
duration: Optional[int] = None
quality: Optional[str] = None
class AudioSearchError(Exception):
"""Base exception for audio search errors."""
pass
class NoResultsError(AudioSearchError):
"""Raised when no search results are found."""
pass
class DownloadError(AudioSearchError):
"""Raised when download fails."""
pass
class AudioSearchService:
"""
Service for searching and downloading audio files.
Service for searching and downloading audio files using flacfetch.
This is a thin wrapper around karaoke_gen.audio_fetcher.FlacFetcher that adds
backend-specific functionality:
- Caches raw results for API-based selection (client picks by index)
- Provides Firestore-compatible serialization
- Manages service lifecycle as singleton
- Remote flacfetch service integration for torrent downloads
When FLACFETCH_API_URL is configured, this service uses the remote flacfetch
HTTP API for search and download operations. This enables torrent downloads
which are not possible in Cloud Run due to network restrictions.
The actual flacfetch integration is in karaoke_gen.audio_fetcher.FlacFetcher,
which is shared between local CLI and cloud backend.
This service is used by the backend to:
1. Search for audio by artist/title
2. Return results for user selection
3. Download the selected audio to a local file
"""
# Sentinel value to indicate "use environment variable"
@@ -81,42 +93,60 @@ class AudioSearchService:
"""
# Use environment variables if not explicitly provided
if redacted_api_key is self._USE_ENV:
redacted_api_key = os.environ.get("REDACTED_API_KEY")
self._redacted_api_key = os.environ.get("REDACTED_API_KEY")
else:
self._redacted_api_key = redacted_api_key
if ops_api_key is self._USE_ENV:
ops_api_key = os.environ.get("OPS_API_KEY")
# Check for remote flacfetch client
self._remote_client = get_flacfetch_client()
# Log which mode we're using
if self._remote_client:
logger.info("AudioSearchService using REMOTE flacfetch service (torrent downloads enabled)")
self._ops_api_key = os.environ.get("OPS_API_KEY")
else:
logger.info("AudioSearchService using LOCAL flacfetch (YouTube only in Cloud Run)")
self._ops_api_key = ops_api_key
self._manager = None
self._raw_results = {} # Cache raw results for download
def _get_manager(self):
"""Lazily initialize and return the FetchManager."""
if self._manager is None:
try:
from flacfetch.core.manager import FetchManager
from flacfetch.providers.youtube import YouTubeProvider
except ImportError as e:
logger.error(f"flacfetch not installed: {e}")
raise AudioSearchError(
"flacfetch library is not installed. "
"Install with: pip install flacfetch"
) from e
self._manager = FetchManager()
# Add providers based on available API keys
if self._redacted_api_key:
try:
from flacfetch.providers.redacted import RedactedProvider
self._manager.add_provider(RedactedProvider(api_key=self._redacted_api_key))
logger.debug("Added Redacted provider")
except ImportError:
logger.warning("Redacted provider not available")
if self._ops_api_key:
try:
from flacfetch.providers.ops import OPSProvider
self._manager.add_provider(OPSProvider(api_key=self._ops_api_key))
logger.debug("Added OPS provider")
except ImportError:
logger.warning("OPS provider not available")
# Always add YouTube as a fallback provider
self._manager.add_provider(YouTubeProvider())
logger.debug("Added YouTube provider")
# Delegate to shared FlacFetcher implementation (for local mode or fallback)
self._fetcher = FlacFetcher(
redacted_api_key=redacted_api_key,
ops_api_key=ops_api_key,
)
# Cache search results for API-based selection flow
# Key: index, Value: AudioSearchResult (with raw_result)
self._cached_results: List[AudioSearchResult] = []
# Cache for remote search results (search_id -> results)
self._remote_search_id: Optional[str] = None
return self._manager
def search(self, artist: str, title: str) -> List[AudioSearchResult]:
"""
Search for audio matching the given artist and title.
Results are cached internally for later download via download().
If a remote flacfetch service is configured (FLACFETCH_API_URL), uses that
for better torrent download support. Otherwise falls back to local flacfetch.
Args:
artist: The artist name to search for
title: The track title to search for
@@ -126,90 +156,49 @@ class AudioSearchService:
Raises:
NoResultsError: If no results are found
AudioSearchError: For other errors (e.g., flacfetch not installed)
AudioSearchError: For other errors
"""
try:
# Try remote flacfetch service first if configured
if self._remote_client:
return self._search_remote(artist, title)
# Fallback to local flacfetch
results = self._fetcher.search(artist, title)
# Cache results for later download
self._cached_results = results
self._remote_search_id = None
logger.info(f"Found {len(results)} results for: {artist} - {title}")
return results
except (NoResultsError, AudioFetcherError):
# Re-raise these as-is
raise
from flacfetch.core.models import TrackQuery
except ImportError as e:
raise AudioSearchError("flacfetch not installed") from e
manager = self._get_manager()
query = TrackQuery(artist=artist, title=title)
logger.info(f"Searching for audio: {artist} - {title}")
try:
results = manager.search(query)
except Exception as e:
logger.error(f"Search failed: {e}")
raise AudioSearchError(f"Search failed: {e}") from e
def _search_remote(self, artist: str, title: str) -> List[AudioSearchResult]:
"""
Search using the remote flacfetch service.
Runs async code in a sync context for compatibility with existing API.
"""
try:
# Run async search in sync context
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
response = loop.run_until_complete(
self._remote_client.search(artist, title)
)
finally:
loop.close()
if not results:
raise NoResultsError(f"No results found for: {artist} - {title}")
# Convert to our AudioSearchResult format and cache raw results
search_results = []
self._raw_results.clear()
for i, result in enumerate(results):
search_result = AudioSearchResult(
title=getattr(result, "title", title),
artist=getattr(result, "artist", artist),
provider=getattr(result, "provider", "Unknown"),
url=getattr(result, "url", ""),
duration=getattr(result, "duration", None),
quality=getattr(result, "quality", None),
source_id=getattr(result, "id", str(i)),
index=i,
)
search_results.append(search_result)
# Check for empty results
if not response.get("results"):
raise NoResultsError(f"No results found for: {artist} - {title}")
# Store search_id for download
self._remote_search_id = response.get("search_id")
# Convert remote results to AudioSearchResult objects
results = []
for item in response.get("results", []):
result = AudioSearchResult(
title=item.get("title", "Unknown"),
artist=item.get("artist", "Unknown"),
provider=item.get("provider", "Unknown"),
quality=item.get("quality", ""),
size_bytes=item.get("size_bytes"),
duration_seconds=item.get("duration_seconds"),
is_lossless=item.get("is_lossless", False),
seeders=item.get("seeders"),
year=item.get("year"),
extra_info={
"index": item.get("index"),
"target_file": item.get("target_file"),
"view_count": item.get("view_count"),
"channel": item.get("channel"),
"remote": True, # Flag for download logic
},
)
results.append(result)
# Cache results
self._cached_results = results
logger.info(f"Found {len(results)} results from remote flacfetch for: {artist} - {title}")
return results
except FlacfetchServiceError as e:
logger.warning(f"Remote flacfetch search failed, falling back to local: {e}")
# Fallback to local search
self._remote_search_id = None
results = self._fetcher.search(artist, title)
self._cached_results = results
return results
# Cache raw result for later download
self._raw_results[i] = result
logger.info(f"Found {len(search_results)} results for: {artist} - {title}")
return search_results
def select_best(self, results: List[AudioSearchResult]) -> int:
"""
@@ -223,136 +212,90 @@ class AudioSearchService:
Returns:
Index of the best result
"""
return self._fetcher.select_best(results)
if not results:
return 0
manager = self._get_manager()
# Get raw results for the indices we have
raw_results = [self._raw_results.get(r.index) for r in results if r.index in self._raw_results]
if raw_results:
try:
best = manager.select_best(raw_results)
# Find index of best result
for i, raw in enumerate(raw_results):
if raw == best:
return results[i].index
except Exception as e:
logger.warning(f"select_best failed, using first result: {e}")
# Fallback: return first result
return results[0].index if results else 0
def download(
self,
result_index: int,
output_dir: str,
output_filename: Optional[str] = None,
gcs_path: Optional[str] = None,
) -> AudioDownloadResult:
"""
Download audio from a cached search result.
This method uses the cached results from the last search() call.
The API flow is:
1. Client calls search() -> gets list of results
2. Client picks an index
3. Client calls download(index) -> gets downloaded file
If a remote flacfetch service is configured and the search was performed
remotely (for torrent sources), uses the remote service for download.
Download audio from a search result.
Args:
result_index: Index of the result to download (from search results)
output_dir: Directory to save the downloaded file
output_filename: Optional filename (without extension)
gcs_path: Optional GCS path for remote uploads (e.g., "uploads/job123/audio/")
Returns:
AudioDownloadResult with the downloaded file path
Raises:
DownloadError: If download fails or no cached result for index
DownloadError: If download fails
"""
if result_index < 0 or result_index >= len(self._cached_results):
raise DownloadError(
f"No cached result for index {result_index}. "
f"Available indices: 0-{len(self._cached_results) - 1}. "
"Run search() first."
)
if result_index not in self._raw_results:
raise DownloadError(f"No cached result for index {result_index}. Run search() first.")
result = self._cached_results[result_index]
raw_result = self._raw_results[result_index]
manager = self._get_manager()
logger.info(f"Downloading: {result.artist} - {result.title} from {result.provider}")
# Ensure output directory exists
os.makedirs(output_dir, exist_ok=True)
# Check if this was a remote search (torrent sources need remote download)
if self._remote_search_id and self._remote_client:
# Check if this is a torrent source that needs remote handling
is_remote = result.extra_info.get("remote", False) if result.extra_info else False
is_torrent = result.provider in ["Redacted", "OPS"]
if is_remote or is_torrent:
return self._download_remote(result_index, output_dir, output_filename, gcs_path)
# Get metadata from raw result
artist = getattr(raw_result, "artist", "Unknown")
title = getattr(raw_result, "title", "Unknown")
provider = getattr(raw_result, "provider", "Unknown")
# Delegate to shared FlacFetcher (local download)
fetch_result = self._fetcher.download(result, output_dir, output_filename)
# Generate filename if not provided
if output_filename is None:
output_filename = f"{artist} - {title}"
logger.info(f"Downloaded to: {fetch_result.filepath}")
return fetch_result
def _download_remote(
self,
result_index: int,
output_dir: str,
output_filename: Optional[str] = None,
gcs_path: Optional[str] = None,
) -> AudioDownloadResult:
"""
Download using the remote flacfetch service.
The remote service downloads via torrent and optionally uploads to GCS.
"""
if not self._remote_search_id:
raise DownloadError("No remote search ID - run search() first")
result = self._cached_results[result_index]
logger.info(f"Downloading: {artist} - {title} from {provider}")
try:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
try:
# Start download
download_id = loop.run_until_complete(
self._remote_client.download(
search_id=self._remote_search_id,
result_index=result_index,
output_filename=output_filename,
gcs_path=gcs_path,
)
)
logger.info(f"Remote download started: {download_id}")
# Wait for completion with progress logging
def log_progress(status):
progress = status.get("progress", 0)
speed = status.get("download_speed_kbps", 0)
logger.debug(f"Download progress: {progress:.1f}% ({speed:.1f} KB/s)")
final_status = loop.run_until_complete(
self._remote_client.wait_for_download(
download_id,
timeout=600, # 10 minute timeout
progress_callback=log_progress,
)
)
finally:
loop.close()
filepath = manager.download(
raw_result,
output_path=output_dir,
output_filename=output_filename,
)
# Determine file path
filepath = final_status.get("gcs_path") or final_status.get("output_path")
if filepath is None:
raise DownloadError(f"Download returned no file path for: {artist} - {title}")
if not filepath:
raise DownloadError("Remote download completed but no file path returned")
logger.info(f"Remote download complete: {filepath}")
logger.info(f"Downloaded to: {filepath}")
return AudioDownloadResult(
filepath=filepath,
artist=result.artist,
title=result.title,
provider=result.provider,
quality=result.quality,
artist=artist,
title=title,
provider=provider,
duration=getattr(raw_result, "duration", None),
quality=getattr(raw_result, "quality", None),
)
except FlacfetchServiceError as e:
raise DownloadError(f"Remote download failed: {e}") from e
except Exception as e:
logger.error(f"Remote download error: {e}")
raise DownloadError(f"Remote download failed: {e}") from e
raise DownloadError(f"Failed to download {artist} - {title}: {e}") from e
def search_and_download_auto(
self,
@@ -360,7 +303,6 @@ class AudioSearchService:
title: str,
output_dir: str,
output_filename: Optional[str] = None,
gcs_path: Optional[str] = None,
) -> AudioDownloadResult:
"""
Search for audio and automatically download the best result.
@@ -373,7 +315,6 @@ class AudioSearchService:
title: The track title to search for
output_dir: Directory to save the downloaded file
output_filename: Optional filename (without extension)
gcs_path: Optional GCS path for remote uploads
Returns:
AudioDownloadResult with the downloaded file path
@@ -387,64 +328,10 @@ class AudioSearchService:
# Select best
best_index = self.select_best(results)
best_result = results[best_index]
logger.info(
f"Auto-selected result {best_index}: "
f"{best_result.artist} - {best_result.title} ({best_result.provider})"
)
logger.info(f"Auto-selected result {best_index}: {results[best_index].artist} - {results[best_index].title} ({results[best_index].provider})")
# Download
return self.download(best_index, output_dir, output_filename, gcs_path)
async def search_async(self, artist: str, title: str) -> List[AudioSearchResult]:
"""
Async version of search for use in async routes.
Note: Currently wraps sync search in executor. Future optimization
could make this fully async.
"""
loop = asyncio.get_event_loop()
return await loop.run_in_executor(None, self.search, artist, title)
async def download_async(
self,
result_index: int,
output_dir: str,
output_filename: Optional[str] = None,
gcs_path: Optional[str] = None,
) -> AudioDownloadResult:
"""
Async version of download for use in async routes.
Note: Currently wraps sync download in executor. Future optimization
could make this fully async.
"""
loop = asyncio.get_event_loop()
return await loop.run_in_executor(
None,
lambda: self.download(result_index, output_dir, output_filename, gcs_path)
)
def is_remote_enabled(self) -> bool:
"""Check if remote flacfetch service is configured."""
return self._remote_client is not None
async def check_remote_health(self) -> Optional[dict]:
"""
Check health of remote flacfetch service.
Returns:
Health status dict if remote service is configured and healthy,
None if not configured or unhealthy.
"""
if not self._remote_client:
return None
try:
return await self._remote_client.health_check()
except Exception as e:
logger.warning(f"Remote flacfetch health check failed: {e}")
return None
return self.download(best_index, output_dir, output_filename)
# Singleton instance
@@ -457,3 +344,5 @@ def get_audio_search_service() -> AudioSearchService:
if _audio_search_service is None:
_audio_search_service = AudioSearchService()
return _audio_search_service
-412
View File
@@ -1,412 +0,0 @@
"""
Client for the remote flacfetch HTTP API service.
This client communicates with a dedicated flacfetch VM that handles:
- BitTorrent downloads from private trackers (Redacted, OPS)
- YouTube downloads
- GCS uploads of downloaded files
The flacfetch service provides:
- Full peer connectivity for torrents (not possible in Cloud Run)
- Indefinite seeding of completed torrents
- Automatic disk cleanup
Usage:
client = get_flacfetch_client()
if client:
# Search for audio
search_result = await client.search("ABBA", "Waterloo")
# Download the best result
download_id = await client.download(
search_id=search_result["search_id"],
result_index=0,
gcs_path="uploads/job123/audio/",
)
# Wait for completion
result = await client.wait_for_download(download_id)
print(f"Downloaded to: {result['gcs_path']}")
"""
import asyncio
import logging
from typing import Any, Dict, List, Optional
import httpx
logger = logging.getLogger(__name__)
class FlacfetchServiceError(Exception):
"""Error communicating with or returned from flacfetch service."""
pass
class FlacfetchClient:
"""
Client for remote flacfetch HTTP API.
All endpoints require authentication via X-API-Key header.
"""
def __init__(
self,
base_url: str,
api_key: str,
timeout: int = 60,
):
"""
Initialize flacfetch client.
Args:
base_url: Base URL of flacfetch service (e.g., http://10.0.0.5:8080)
api_key: API key for authentication
timeout: Default request timeout in seconds
"""
self.base_url = base_url.rstrip('/')
self.api_key = api_key
self.timeout = timeout
logger.info(f"FlacfetchClient initialized with base_url={self.base_url}")
def _headers(self) -> Dict[str, str]:
"""Get request headers with authentication."""
return {
"X-API-Key": self.api_key,
"Content-Type": "application/json",
}
async def health_check(self) -> Dict[str, Any]:
"""
Check if flacfetch service is healthy.
Returns:
Health status dict with transmission, disk, and provider info
Raises:
FlacfetchServiceError: If service is unhealthy or unreachable
"""
try:
async with httpx.AsyncClient() as client:
resp = await client.get(
f"{self.base_url}/health",
headers=self._headers(),
timeout=10,
)
resp.raise_for_status()
data = resp.json()
if data.get("status") not in ["healthy", "degraded"]:
raise FlacfetchServiceError(f"Service unhealthy: {data}")
return data
except httpx.RequestError as e:
raise FlacfetchServiceError(f"Cannot reach flacfetch service: {e}")
except httpx.HTTPStatusError as e:
raise FlacfetchServiceError(f"Health check failed: {e.response.status_code}")
async def search(
self,
artist: str,
title: str,
) -> Dict[str, Any]:
"""
Search for audio matching artist and title.
Args:
artist: Artist name
title: Track title
Returns:
Search response dict with search_id and results list
Raises:
FlacfetchServiceError: On search failure
"""
try:
async with httpx.AsyncClient() as client:
resp = await client.post(
f"{self.base_url}/search",
headers=self._headers(),
json={"artist": artist, "title": title},
timeout=self.timeout,
)
if resp.status_code == 404:
# No results found - return empty results
return {
"search_id": None,
"artist": artist,
"title": title,
"results": [],
"results_count": 0,
}
resp.raise_for_status()
return resp.json()
except httpx.RequestError as e:
raise FlacfetchServiceError(f"Search request failed: {e}")
except httpx.HTTPStatusError as e:
raise FlacfetchServiceError(f"Search failed: {e.response.status_code} - {e.response.text}")
async def download(
self,
search_id: str,
result_index: int,
output_filename: Optional[str] = None,
gcs_path: Optional[str] = None,
) -> str:
"""
Start downloading an audio file.
Args:
search_id: Search ID from previous search
result_index: Index of result to download
output_filename: Optional custom filename (without extension)
gcs_path: GCS path for upload (e.g., "uploads/job123/audio/")
Returns:
Download ID for tracking progress
Raises:
FlacfetchServiceError: On download start failure
"""
try:
async with httpx.AsyncClient() as client:
payload = {
"search_id": search_id,
"result_index": result_index,
}
if output_filename:
payload["output_filename"] = output_filename
if gcs_path:
payload["upload_to_gcs"] = True
payload["gcs_path"] = gcs_path
resp = await client.post(
f"{self.base_url}/download",
headers=self._headers(),
json=payload,
timeout=self.timeout,
)
resp.raise_for_status()
data = resp.json()
return data["download_id"]
except httpx.RequestError as e:
raise FlacfetchServiceError(f"Download request failed: {e}")
except httpx.HTTPStatusError as e:
raise FlacfetchServiceError(f"Download start failed: {e.response.status_code} - {e.response.text}")
async def get_download_status(self, download_id: str) -> Dict[str, Any]:
"""
Get the current status of a download.
Args:
download_id: Download ID from start_download
Returns:
Status dict with progress, speed, path info
Raises:
FlacfetchServiceError: On status check failure
"""
try:
async with httpx.AsyncClient() as client:
resp = await client.get(
f"{self.base_url}/download/{download_id}/status",
headers=self._headers(),
timeout=10,
)
resp.raise_for_status()
return resp.json()
except httpx.RequestError as e:
raise FlacfetchServiceError(f"Status request failed: {e}")
except httpx.HTTPStatusError as e:
if e.response.status_code == 404:
raise FlacfetchServiceError(f"Download not found: {download_id}")
raise FlacfetchServiceError(f"Status check failed: {e.response.status_code}")
async def wait_for_download(
self,
download_id: str,
timeout: int = 600,
poll_interval: float = 2.0,
progress_callback: Optional[callable] = None,
) -> Dict[str, Any]:
"""
Wait for a download to complete.
Args:
download_id: Download ID to wait for
timeout: Maximum wait time in seconds
poll_interval: Time between status checks
progress_callback: Optional callback(status_dict) for progress updates
Returns:
Final status dict with output_path or gcs_path
Raises:
FlacfetchServiceError: On download failure or timeout
"""
elapsed = 0.0
while elapsed < timeout:
status = await self.get_download_status(download_id)
if progress_callback:
try:
progress_callback(status)
except Exception as e:
logger.warning(f"Progress callback error: {e}")
download_status = status.get("status")
if download_status in ["complete", "seeding"]:
logger.info(f"Download {download_id} complete: {status.get('gcs_path') or status.get('output_path')}")
return status
elif download_status == "failed":
error = status.get("error", "Unknown error")
raise FlacfetchServiceError(f"Download failed: {error}")
elif download_status == "cancelled":
raise FlacfetchServiceError("Download was cancelled")
# Log progress
progress = status.get("progress", 0)
speed = status.get("download_speed_kbps", 0)
peers = status.get("peers", 0)
logger.debug(
f"Download {download_id}: {progress:.1f}% "
f"({speed:.1f} KB/s, {peers} peers)"
)
await asyncio.sleep(poll_interval)
elapsed += poll_interval
raise FlacfetchServiceError(f"Download timed out after {timeout}s")
async def list_torrents(self) -> Dict[str, Any]:
"""
List all torrents in Transmission.
Returns:
Torrent list response with torrent info and totals
"""
try:
async with httpx.AsyncClient() as client:
resp = await client.get(
f"{self.base_url}/torrents",
headers=self._headers(),
timeout=30,
)
resp.raise_for_status()
return resp.json()
except httpx.RequestError as e:
raise FlacfetchServiceError(f"List torrents failed: {e}")
except httpx.HTTPStatusError as e:
raise FlacfetchServiceError(f"List torrents failed: {e.response.status_code}")
async def delete_torrent(
self,
torrent_id: int,
delete_data: bool = True,
) -> Dict[str, Any]:
"""
Delete a torrent from Transmission.
Args:
torrent_id: Transmission torrent ID
delete_data: Also delete downloaded files
Returns:
Delete response
"""
try:
async with httpx.AsyncClient() as client:
resp = await client.delete(
f"{self.base_url}/torrents/{torrent_id}",
headers=self._headers(),
params={"delete_data": str(delete_data).lower()},
timeout=30,
)
resp.raise_for_status()
return resp.json()
except httpx.RequestError as e:
raise FlacfetchServiceError(f"Delete torrent failed: {e}")
except httpx.HTTPStatusError as e:
raise FlacfetchServiceError(f"Delete torrent failed: {e.response.status_code}")
async def cleanup_torrents(
self,
strategy: str = "oldest",
target_free_gb: float = 10.0,
) -> Dict[str, Any]:
"""
Trigger disk cleanup.
Args:
strategy: Cleanup strategy (oldest, largest, lowest_ratio)
target_free_gb: Target free space to achieve
Returns:
Cleanup response with removed count and freed space
"""
try:
async with httpx.AsyncClient() as client:
resp = await client.post(
f"{self.base_url}/torrents/cleanup",
headers=self._headers(),
json={
"strategy": strategy,
"target_free_gb": target_free_gb,
},
timeout=60,
)
resp.raise_for_status()
return resp.json()
except httpx.RequestError as e:
raise FlacfetchServiceError(f"Cleanup failed: {e}")
except httpx.HTTPStatusError as e:
raise FlacfetchServiceError(f"Cleanup failed: {e.response.status_code}")
# Singleton client instance
_client: Optional[FlacfetchClient] = None
def get_flacfetch_client() -> Optional[FlacfetchClient]:
"""
Get the flacfetch client if configured.
Returns:
FlacfetchClient instance if FLACFETCH_API_URL and FLACFETCH_API_KEY are set,
None otherwise (indicating local-only mode).
"""
global _client
if _client is None:
from backend.config import get_settings
settings = get_settings()
if settings.flacfetch_api_url and settings.flacfetch_api_key:
_client = FlacfetchClient(
base_url=settings.flacfetch_api_url,
api_key=settings.flacfetch_api_key,
)
logger.info("FlacfetchClient initialized for remote service")
else:
logger.debug("FlacfetchClient not configured (missing URL or API key)")
return _client
def reset_flacfetch_client():
"""Reset the singleton client (for testing)."""
global _client
_client = None
+93 -206
View File
@@ -99,11 +99,7 @@ class TestJobStatusAudioSearchStates:
class TestAudioSearchResult:
"""Test AudioSearchResult dataclass.
AudioSearchResult is now imported from karaoke_gen.audio_fetcher.
These tests verify the backend can use it correctly.
"""
"""Test AudioSearchResult dataclass."""
def test_create_audio_search_result(self):
"""Test creating an AudioSearchResult."""
@@ -144,8 +140,6 @@ class TestAudioSearchResult:
assert data['provider'] == "YouTube"
assert data['url'] == "https://youtube.com/watch?v=abc123"
assert data['index'] == 0
# raw_result should NOT be in serialized dict
assert 'raw_result' not in data
def test_from_dict(self):
"""Test creating from dict."""
@@ -166,7 +160,6 @@ class TestAudioSearchResult:
assert result.artist == "ABBA"
assert result.provider == "YouTube"
assert result.index == 0
assert result.raw_result is None # Not set from dict
class TestAudioSearchServiceInit:
@@ -178,9 +171,10 @@ class TestAudioSearchServiceInit:
# Pass explicit None to override environment
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
# Service should have a FlacFetcher internally
assert service._fetcher is not None
assert service._cached_results == []
# When explicitly passed None, it should be None (not from env)
assert service._redacted_api_key is None
assert service._ops_api_key is None
assert service._manager is None
def test_init_with_keys(self):
"""Test initialization with API keys."""
@@ -189,40 +183,41 @@ class TestAudioSearchServiceInit:
ops_api_key="test_ops_key",
)
# Keys are passed to FlacFetcher
assert service._fetcher._redacted_api_key == "test_redacted_key"
assert service._fetcher._ops_api_key == "test_ops_key"
assert service._redacted_api_key == "test_redacted_key"
assert service._ops_api_key == "test_ops_key"
def test_init_reads_from_environment(self):
"""Test initialization reads keys from environment variables."""
import os
# Just test that the service can be initialized
service = AudioSearchService()
# The service should have a FlacFetcher
assert service._fetcher is not None
# The service should have API keys from env or None
assert service._manager is None # Not yet initialized
class TestAudioSearchServiceSearch:
"""Test AudioSearchService.search() method."""
def test_search_returns_results(self):
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
def test_search_returns_results(self, mock_get_manager):
"""Test search returns AudioSearchResult list."""
# Create mock result
mock_result = AudioSearchResult(
title="Waterloo",
artist="ABBA",
provider="YouTube",
url="https://youtube.com/watch?v=abc123",
duration=180,
quality="FLAC",
source_id="abc123",
index=0,
)
# Mock the flacfetch result
mock_result = Mock()
mock_result.title = "Waterloo"
mock_result.artist = "ABBA"
mock_result.provider = "YouTube"
mock_result.url = "https://youtube.com/watch?v=abc123"
mock_result.duration = 180
mock_result.quality = "FLAC"
mock_result.id = "abc123"
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
service._fetcher = Mock()
service._fetcher.search.return_value = [mock_result]
mock_manager = Mock()
mock_manager.search.return_value = [mock_result]
mock_get_manager.return_value = mock_manager
service = AudioSearchService()
results = service.search("ABBA", "Waterloo")
assert len(results) == 1
@@ -231,36 +226,40 @@ class TestAudioSearchServiceSearch:
assert results[0].provider == "YouTube"
assert results[0].index == 0
def test_search_no_results_raises_error(self):
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
def test_search_no_results_raises_error(self, mock_get_manager):
"""Test search raises NoResultsError when no results."""
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
service._fetcher = Mock()
service._fetcher.search.side_effect = NoResultsError("No results found")
mock_manager = Mock()
mock_manager.search.return_value = []
mock_get_manager.return_value = mock_manager
service = AudioSearchService()
with pytest.raises(NoResultsError) as exc_info:
service.search("Unknown Artist", "Unknown Song")
assert "No results found" in str(exc_info.value)
def test_search_multiple_results(self):
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
def test_search_multiple_results(self, mock_get_manager):
"""Test search returns multiple results with correct indices."""
mock_results = []
for i in range(3):
mock_results.append(AudioSearchResult(
title=f"Song {i}",
artist="Artist",
provider=["YouTube", "Redacted", "OPS"][i],
url=f"https://example.com/{i}",
duration=180 + i * 10,
quality=["320kbps", "FLAC", "FLAC"][i],
source_id=str(i),
index=i,
))
mock_result = Mock()
mock_result.title = f"Song {i}"
mock_result.artist = "Artist"
mock_result.provider = ["YouTube", "Redacted", "OPS"][i]
mock_result.url = f"https://example.com/{i}"
mock_result.duration = 180 + i * 10
mock_result.quality = ["320kbps", "FLAC", "FLAC"][i]
mock_result.id = str(i)
mock_results.append(mock_result)
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
service._fetcher = Mock()
service._fetcher.search.return_value = mock_results
mock_manager = Mock()
mock_manager.search.return_value = mock_results
mock_get_manager.return_value = mock_manager
service = AudioSearchService()
results = service.search("Artist", "Song")
assert len(results) == 3
@@ -272,39 +271,34 @@ class TestAudioSearchServiceSearch:
class TestAudioSearchServiceSelectBest:
"""Test AudioSearchService.select_best() method."""
def test_select_best_delegates_to_fetcher(self):
"""Test select_best uses FlacFetcher's select_best."""
mock_results = [
AudioSearchResult(
title="Waterloo",
artist="ABBA",
provider="YouTube",
url="https://youtube.com/watch?v=abc123",
index=0,
),
AudioSearchResult(
title="Waterloo",
artist="ABBA",
provider="Redacted",
url="https://example.com/456",
index=1,
),
]
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
def test_select_best_uses_manager(self, mock_get_manager):
"""Test select_best uses flacfetch's select_best."""
# First need to do a search to populate cache
mock_result = Mock()
mock_result.title = "Waterloo"
mock_result.artist = "ABBA"
mock_result.provider = "YouTube"
mock_result.url = "https://youtube.com/watch?v=abc123"
mock_result.duration = 180
mock_result.quality = "FLAC"
mock_result.id = "abc123"
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
service._fetcher = Mock()
service._fetcher.select_best.return_value = 1
mock_manager = Mock()
mock_manager.search.return_value = [mock_result]
mock_manager.select_best.return_value = mock_result
mock_get_manager.return_value = mock_manager
best_index = service.select_best(mock_results)
service = AudioSearchService()
results = service.search("ABBA", "Waterloo")
assert best_index == 1
service._fetcher.select_best.assert_called_once_with(mock_results)
best_index = service.select_best(results)
assert best_index == 0
def test_select_best_empty_list_returns_zero(self):
"""Test select_best returns 0 for empty list."""
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
service._fetcher = Mock()
service._fetcher.select_best.return_value = 0
service = AudioSearchService()
best_index = service.select_best([])
@@ -314,18 +308,24 @@ class TestAudioSearchServiceSelectBest:
class TestAudioSearchServiceDownload:
"""Test AudioSearchService.download() method."""
def test_download_without_search_raises_error(self):
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
def test_download_without_search_raises_error(self, mock_get_manager):
"""Test download without prior search raises error."""
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
mock_manager = Mock()
mock_get_manager.return_value = mock_manager
service = AudioSearchService()
with pytest.raises(DownloadError) as exc_info:
service.download(0, "/tmp")
assert "No cached result" in str(exc_info.value)
def test_download_after_search(self):
@patch('backend.services.audio_search_service.AudioSearchService._get_manager')
def test_download_after_search(self, mock_get_manager):
"""Test download after search works."""
import tempfile
import os
# Create a temp file to simulate downloaded file
temp_dir = tempfile.mkdtemp()
@@ -333,32 +333,23 @@ class TestAudioSearchServiceDownload:
with open(temp_file, 'w') as f:
f.write("test")
mock_result = AudioSearchResult(
title="Waterloo",
artist="ABBA",
provider="YouTube",
url="https://youtube.com/watch?v=abc123",
duration=180,
quality="FLAC",
source_id="abc123",
index=0,
)
mock_result = Mock()
mock_result.title = "Waterloo"
mock_result.artist = "ABBA"
mock_result.provider = "YouTube"
mock_result.url = "https://youtube.com/watch?v=abc123"
mock_result.duration = 180
mock_result.quality = "FLAC"
mock_result.id = "abc123"
mock_fetch_result = AudioDownloadResult(
filepath=temp_file,
artist="ABBA",
title="Waterloo",
provider="YouTube",
duration=180,
quality="FLAC",
)
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
service._fetcher = Mock()
service._fetcher.search.return_value = [mock_result]
service._fetcher.download.return_value = mock_fetch_result
mock_manager = Mock()
mock_manager.search.return_value = [mock_result]
mock_manager.download.return_value = temp_file
mock_get_manager.return_value = mock_manager
service = AudioSearchService()
service.search("ABBA", "Waterloo")
result = service.download(0, temp_dir)
assert result.filepath == temp_file
@@ -369,27 +360,6 @@ class TestAudioSearchServiceDownload:
# Cleanup
os.remove(temp_file)
os.rmdir(temp_dir)
def test_download_invalid_index_raises_error(self):
"""Test download with invalid index raises error."""
mock_result = AudioSearchResult(
title="Waterloo",
artist="ABBA",
provider="YouTube",
url="https://youtube.com/watch?v=abc123",
index=0,
)
service = AudioSearchService(redacted_api_key=None, ops_api_key=None)
service._fetcher = Mock()
service._fetcher.search.return_value = [mock_result]
service.search("ABBA", "Waterloo")
with pytest.raises(DownloadError) as exc_info:
service.download(99, "/tmp")
assert "No cached result for index 99" in str(exc_info.value)
class TestGetAudioSearchService:
@@ -414,86 +384,3 @@ class TestGetAudioSearchService:
assert service1 is service2
class TestFlacfetchIntegration:
"""
Integration tests that verify flacfetch imports work correctly.
These tests ensure the backend code is compatible with the installed
flacfetch version and would catch issues like renamed classes.
Note: Some tests are marked with pytest.importorskip for flacfetch
since it may not be installed in all test environments (e.g., CI).
"""
def test_flacfetch_imports_work(self):
"""Test that all flacfetch imports in audio_search_service work.
This test would have caught the YouTubeProvider -> YoutubeProvider rename.
"""
flacfetch = pytest.importorskip("flacfetch")
# These imports match what karaoke_gen.audio_fetcher does
from flacfetch.core.manager import FetchManager
from flacfetch.providers.youtube import YoutubeProvider
from flacfetch.core.models import TrackQuery
# Verify the classes exist and are callable
assert FetchManager is not None
assert YoutubeProvider is not None
assert TrackQuery is not None
def test_karaoke_gen_audio_fetcher_imports_work(self):
"""Test that karaoke_gen.audio_fetcher imports work.
Since backend now imports from karaoke_gen, this is the critical test.
This should work even without flacfetch installed (lazy import).
"""
from karaoke_gen.audio_fetcher import (
FlacFetcher,
AudioSearchResult,
AudioFetchResult,
AudioFetcherError,
NoResultsError,
DownloadError,
)
assert FlacFetcher is not None
assert AudioSearchResult is not None
assert AudioFetchResult is not None
assert AudioFetcherError is not None
assert NoResultsError is not None
assert DownloadError is not None
def test_audio_search_service_can_initialize_fetcher(self):
"""Test AudioSearchService can initialize its FlacFetcher.
This verifies the actual initialization code path works.
"""
pytest.importorskip("flacfetch")
service = AudioSearchService(
redacted_api_key=None,
ops_api_key=None,
)
# Verify FlacFetcher was initialized
assert service._fetcher is not None
# FlacFetcher can initialize its manager (tests actual flacfetch)
manager = service._fetcher._get_manager()
assert manager is not None
def test_shared_classes_are_same_as_karaoke_gen(self):
"""Verify backend uses the same classes as karaoke_gen."""
from karaoke_gen.audio_fetcher import (
AudioSearchResult as KGAudioSearchResult,
AudioFetchResult as KGAudioFetchResult,
NoResultsError as KGNoResultsError,
DownloadError as KGDownloadError,
)
# These should be the exact same classes, not copies
assert AudioSearchResult is KGAudioSearchResult
assert AudioDownloadResult is KGAudioFetchResult
assert NoResultsError is KGNoResultsError
assert DownloadError is KGDownloadError
+8 -9
View File
@@ -1,23 +1,22 @@
# Cloud Build configuration for base image
#
# This builds the base image with all Python dependencies pre-installed.
# Run this manually when poetry.lock OR Dockerfile.base changes:
# Run this manually when poetry.lock changes:
# gcloud builds submit --config=cloudbuild-base.yaml --project=nomadkaraoke
#
# The base image is used by the main Dockerfile for fast app builds.
# A combined hash of poetry.lock + Dockerfile.base is stored as a label for change detection.
# The poetry.lock hash is stored as a label for change detection.
steps:
# Calculate combined hash for change detection (poetry.lock + Dockerfile.base)
# Calculate poetry.lock hash for change detection
- name: 'gcr.io/cloud-builders/docker'
entrypoint: 'bash'
args:
- '-c'
- |
# Hash both files to detect changes in either
CONTENT_HASH=$$(cat poetry.lock backend/Dockerfile.base | sha256sum | cut -d' ' -f1)
echo "Combined content hash (poetry.lock + Dockerfile.base): $$CONTENT_HASH"
echo "$$CONTENT_HASH" > /workspace/CONTENT_HASH
POETRY_LOCK_HASH=$$(sha256sum poetry.lock | cut -d' ' -f1)
echo "Poetry.lock hash: $$POETRY_LOCK_HASH"
echo "$$POETRY_LOCK_HASH" > /workspace/POETRY_LOCK_HASH
# Build base image with all dependencies
- name: 'gcr.io/cloud-builders/docker'
@@ -25,11 +24,11 @@ steps:
args:
- '-c'
- |
CONTENT_HASH=$$(cat /workspace/CONTENT_HASH)
POETRY_LOCK_HASH=$$(cat /workspace/POETRY_LOCK_HASH)
docker build \
--build-arg BUILD_DATE=$BUILD_ID \
--build-arg BUILD_ID=$BUILD_ID \
--build-arg BASE_CONTENT_HASH=$$CONTENT_HASH \
--build-arg POETRY_LOCK_HASH=$$POETRY_LOCK_HASH \
-t us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:latest \
-t us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend-base:$BUILD_ID \
-f backend/Dockerfile.base \
+1 -1
View File
@@ -76,7 +76,7 @@ steps:
--max-instances 50 \
--min-instances 0 \
--set-env-vars "GOOGLE_CLOUD_PROJECT=$PROJECT_ID,GCS_BUCKET_NAME=karaoke-gen-storage-$PROJECT_ID,FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,AUDIO_SEPARATOR_API_URL=https://nomadkaraoke--audio-separator-api.modal.run,KARAOKE_GEN_VERSION=$$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1" \
--set-secrets "AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,REDACTED_API_KEY=redacted-api-key:latest,OPS_API_KEY=ops-api-key:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest"
--set-secrets "AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest"
images:
- 'us-central1-docker.pkg.dev/$PROJECT_ID/karaoke-repo/karaoke-backend:latest'
-35
View File
@@ -1,35 +0,0 @@
# Codecov Configuration
# Docs: https://docs.codecov.com/docs/codecovyml-reference
coverage:
# Coverage value precision (2 decimal places)
precision: 2
round: down
# Green range starts at 70% (our target)
range: "70...100"
status:
# Project-level coverage status check
project:
default:
target: 70% # Pass if coverage >= 70%
threshold: 2% # Allow 2% drop without failing
if_ci_failed: error
# Patch coverage (new/changed lines only)
patch:
default:
target: 70% # New code should also meet 70%
threshold: 5% # More lenient for patches
# Disable PR comments entirely since we're satisfied with 70%+ coverage
# Status checks will still appear on the PR
comment: false
# Alternative: Uncomment below to only comment when coverage changes
# comment:
# layout: "diff, flags, files"
# behavior: default
# require_changes: true # Only comment if coverage actually changes
# require_base: true # Require base report to exist
@@ -681,8 +681,6 @@ The karaoke-gen system supports multiple interfaces to the same core functionali
| `dropbox-oauth-credentials` | ✅ | Dropbox API OAuth tokens |
| `gdrive-service-account` | ✅ | Google Drive service account |
| `discord-webhook-url` | ✅ | Default Discord webhook |
| `redacted-api-key` | ⏳ | Redacted tracker API key (flacfetch audio search) |
| `ops-api-key` | ⏳ | OPS/Orpheus tracker API key (flacfetch audio search) |
### Default Distribution Settings
@@ -1,141 +0,0 @@
# Refactor: Unify Audio Fetcher Code
**Created:** 2025-12-20
**Completed:** 2025-12-20
**Priority:** MEDIUM (prevents future bugs like YouTubeProvider -> YoutubeProvider)
**Status:** ✅ Completed
## Problem
The codebase has **two separate implementations** of flacfetch integration:
1. `karaoke_gen/audio_fetcher.py` - Used by local CLI
2. `backend/services/audio_search_service.py` - Used by cloud backend
This violates DRY and caused a bug where the backend's import wasn't updated when flacfetch renamed `YouTubeProvider` to `YoutubeProvider`.
## Current State
### Duplicated Code
| Component | Local (`audio_fetcher.py`) | Backend (`audio_search_service.py`) |
|-----------|---------------------------|-------------------------------------|
| `_get_manager()` | ✅ 40 lines | ❌ 35 lines (copy) |
| Provider setup | ✅ Redacted, OPS, YouTube | ❌ Same, diverged |
| `search()` | ✅ Full implementation | ❌ Similar, diverged |
| `download()` | ✅ With downloaders | ⚠️ Different, simpler |
| `AudioSearchResult` | ✅ Dataclass | ❌ Duplicate, different fields |
| `AudioDownloadResult` | ✅ (`AudioFetchResult`) | ❌ Duplicate |
| Exceptions | ✅ 4 types | ❌ 3 types, different names |
### Drift Examples
- Backend missing `raw_result` field on `AudioSearchResult`
- Backend has `index` field, local doesn't
- Backend exceptions inherit from `AudioSearchError`, local from `AudioFetcherError`
- Backend missing `UserCancelledError`
- Backend missing downloader registration (TorrentDownloader, YoutubeDownloader)
## Proposed Solution
### Option 1: Backend imports from karaoke_gen (Recommended)
The backend already depends on `karaoke_gen` (installed via `pip install -e .` in Dockerfile).
```python
# backend/services/audio_search_service.py - REFACTORED
from karaoke_gen.audio_fetcher import (
FlacFetcher,
AudioSearchResult,
AudioFetchResult,
AudioFetcherError,
NoResultsError,
DownloadError,
)
class AudioSearchService:
"""
Thin wrapper around karaoke_gen.audio_fetcher.FlacFetcher.
Adds backend-specific functionality:
- Firestore serialization (to_dict/from_dict)
- GCS integration for downloads
- Job state management
"""
def __init__(
self,
redacted_api_key: Optional[str] = None,
ops_api_key: Optional[str] = None,
):
self._fetcher = FlacFetcher(
redacted_api_key=redacted_api_key,
ops_api_key=ops_api_key,
)
def search(self, artist: str, title: str) -> List[AudioSearchResult]:
"""Delegate to FlacFetcher.search()."""
return self._fetcher.search(artist, title)
def select_best(self, results: List[AudioSearchResult]) -> int:
"""Delegate to FlacFetcher.select_best()."""
return self._fetcher.select_best(results)
def download(
self,
result: AudioSearchResult,
output_dir: str,
output_filename: Optional[str] = None,
) -> AudioFetchResult:
"""Delegate to FlacFetcher.download()."""
return self._fetcher.download(result, output_dir, output_filename)
```
### Changes Required to karaoke_gen.audio_fetcher
1. **Add `to_dict()` method to `AudioSearchResult`** - For Firestore serialization
2. **Add `from_dict()` classmethod** - For deserialization
3. **Ensure `select_best()` is a standalone method** - Currently buried in interactive flow
4. **Export all public classes from `__init__.py`**
### Changes Required to backend
1. **Remove duplicate dataclasses** - Use imports from karaoke_gen
2. **Remove duplicate `_get_manager()`** - Delegate to FlacFetcher
3. **Remove duplicate exception classes** - Use karaoke_gen exceptions
4. **Update tests** - Point to real classes, not mocks
## Benefits
1. **Single source of truth** - Bug fixes apply everywhere
2. **No more drift** - Dataclasses, exceptions stay in sync
3. **Better test coverage** - karaoke_gen has integration tests for flacfetch imports
4. **Less code to maintain** - Backend becomes thin wrapper
## Migration Steps
1. [x] Add `to_dict()` / `from_dict()` to `AudioSearchResult` in karaoke_gen
2. [x] Extract `select_best()` as standalone method in FlacFetcher
3. [x] Update karaoke_gen `__init__.py` exports (with lazy imports)
4. [x] Refactor backend to import from karaoke_gen
5. [x] Update backend tests
6. [x] Delete duplicate code from backend (~150 lines removed)
7. [x] Add integration test that catches import divergence
## Related Files
- `karaoke_gen/audio_fetcher.py` - Source of truth (keep)
- `backend/services/audio_search_service.py` - Refactor to thin wrapper
- `backend/api/routes/audio_search.py` - Update imports
- `backend/tests/test_audio_search.py` - Update tests
## Estimated Effort
- **Time:** 2-4 hours
- **Risk:** Low (well-defined changes, good test coverage)
- **Dependencies:** None
## Related Architecture Docs
- `docs/01-reference/ARCHITECTURE.md` - "LyricsTranscriber Integration" section shows similar pattern
- `docs/00-current-plan/SCALABLE-ARCHITECTURE-PLAN.md` - "Shared pipeline architecture" goal
@@ -1,729 +0,0 @@
# Flacfetch HTTP API Service Implementation Plan
**Created:** 2025-12-20
**Updated:** 2025-12-20
**Status:** In Progress
**Priority:** Medium
**Estimated Effort:** 3-4 days
## Problem Statement
BitTorrent downloads don't work reliably in Cloud Run due to:
- No inbound peer connections (NAT/firewall)
- Ephemeral containers with no stable IP
- DHT peer discovery blocked
- Container timeout limits (max 60 min)
This prevents downloading lossless audio from private trackers (Redacted, OPS) via the remote backend.
## Solution Overview
Add an HTTP API mode to **flacfetch** (our audio fetching library) and deploy it on a dedicated GCP VM with:
- Full BitTorrent peer connectivity via Transmission daemon
- Static IP for tracker whitelist (seedbox-like)
- Indefinite seeding of downloaded torrents
- Automatic disk cleanup when space runs low
- YouTube downloads also handled (single service for all audio acquisition)
This approach provides clean separation of concerns:
- **flacfetch**: Handles ALL audio acquisition (torrents + YouTube)
- **karaoke-gen backend**: Orchestrates karaoke generation pipeline, calls flacfetch API
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ FLACFETCH SERVICE ARCHITECTURE │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ Cloud Run Backend Flacfetch VM (e2-small) │
│ ┌────────────────────┐ ┌─────────────────────────┐ │
│ │ │ │ │ │
│ │ karaoke-gen │ 1. Search │ flacfetch HTTP API │ │
│ │ AudioSearchService │ ────────────▶ │ POST /search │ │
│ │ │ │ │ │
│ │ │ 2. Results │ Providers: │ │
│ │ │ ◀──────────── │ • Redacted (FLAC) │ │
│ │ │ │ • OPS (FLAC) │ │
│ │ │ 3. Download │ • YouTube (fallback) │ │
│ │ │ ────────────▶ │ │ │
│ │ │ │ POST /download │ │
│ │ │ 4. Status │ GET /download/{id} │ │
│ │ │ ◀──────────── │ │ │
│ │ │ (polling) │ Transmission Daemon │ │
│ │ │ │ • Full peer connect │ │
│ │ │ 5. Complete │ • DHT enabled │ │
│ │ │ ◀──────────── │ • Indefinite seeding │ │
│ │ │ (file path) │ │ │
│ └────────────────────┘ └───────────┬─────────────┘ │
│ │ │ │
│ │ │ 6. Upload to GCS │
│ │ │ │
│ ▼ ▼ │
│ ┌─────────────────────────────────────────────────────────────────┐ │
│ │ Google Cloud Storage │ │
│ │ │ │
│ │ uploads/{job_id}/audio/{filename}.flac │ │
│ │ │ │
│ └─────────────────────────────────────────────────────────────────┘ │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
## Key Design Decisions
### 1. HTTP API in flacfetch (not separate service)
- **Why**: Clean separation of concerns, reusable by future projects, single source of truth
- **How**: Add `flacfetch/api/` module with FastAPI, new CLI mode `flacfetch serve`
### 2. Indefinite Seeding
- **Why**: VM is always-on anyway, helps maintain tracker ratio, acts like a seedbox
- **Current behavior**: Torrent removed after download (line 317-321 in torrent.py)
- **New behavior**: Torrent kept seeding, only removed by cleanup or explicit delete
### 3. Static IP
- **Why**: Required for tracker account IP whitelist, consistent for firewall rules
- **Cost**: ~$3/month additional
### 4. Disk Space Management
- **Strategy**: Auto-cleanup oldest torrents when free space < 5GB threshold
- **API**: Manual cleanup endpoints + automatic background cleanup
### 5. Instance Type: e2-small
- **Specs**: 0.5 vCPU, 2GB RAM, ~$12/month
- **Why**: Sufficient for I/O-bound workload, newer C3/N4 require 2+ vCPUs (overkill)
- **Disk**: 30GB SSD (20GB for torrents + headroom)
### 6. Authentication
- **Method**: Static API key in `X-API-Key` header
- **Firewall**: Permissive (allows future web UI), auth enforced at app level
## Implementation Phases
### Phase 1: Add HTTP API to flacfetch
**Location:** `./flacfetch/flacfetch/api/`
**New Directory Structure:**
```
flacfetch/
├── flacfetch/
│ ├── api/ # NEW
│ │ ├── __init__.py
│ │ ├── main.py # FastAPI app entry point
│ │ ├── auth.py # API key authentication
│ │ ├── models.py # Pydantic request/response models
│ │ ├── routes/
│ │ │ ├── __init__.py
│ │ │ ├── search.py # POST /search
│ │ │ ├── download.py # POST /download, GET /download/{id}/status
│ │ │ ├── torrents.py # GET /torrents, DELETE /torrents/{id}
│ │ │ └── health.py # GET /health, GET /disk-usage
│ │ └── services/
│ │ ├── __init__.py
│ │ ├── download_manager.py # Track active downloads
│ │ └── disk_manager.py # Disk cleanup logic
│ ├── core/
│ ├── downloaders/
│ │ └── torrent.py # MODIFIED: optional keep-seeding mode
│ ├── interface/
│ │ └── cli.py # MODIFIED: add 'serve' subcommand
│ └── providers/
└── pyproject.toml # MODIFIED: add fastapi, uvicorn deps
```
**API Endpoints:**
```yaml
# All endpoints require X-API-Key header
# Search for audio
POST /search
Request:
{ "artist": "ABBA", "title": "Waterloo" }
Response:
{
"search_id": "abc123",
"results": [
{
"index": 0,
"title": "Waterloo",
"artist": "ABBA",
"provider": "Redacted",
"quality": "FLAC 16bit CD",
"seeders": 45,
"size_bytes": 31457280,
"target_file": "02. Waterloo.flac",
...
},
...
]
}
# Start download
POST /download
Request:
{
"search_id": "abc123", # From previous search
"result_index": 0, # Which result to download
"output_filename": "ABBA - Waterloo", # Optional
"upload_to_gcs": true, # Upload to GCS when complete
"gcs_path": "uploads/job123/audio/" # Required if upload_to_gcs
}
Response:
{
"download_id": "dl_xyz789",
"status": "queued"
}
# Check download status
GET /download/{download_id}/status
Response:
{
"download_id": "dl_xyz789",
"status": "downloading", # queued, downloading, uploading, seeding, complete, failed
"progress": 45.2,
"peers": 3,
"download_speed_kbps": 1250.5,
"eta_seconds": 120,
"provider": "Redacted",
"title": "ABBA - Waterloo",
"output_path": null, # Set when complete (local path)
"gcs_path": null, # Set when uploaded to GCS
"error": null
}
# List all torrents (for management)
GET /torrents
Response:
{
"torrents": [
{
"id": 1,
"name": "ABBA - Waterloo [FLAC]",
"status": "seeding",
"progress": 100.0,
"size_bytes": 314572800,
"uploaded_bytes": 628145600, # Seeding stats
"ratio": 2.0,
"added_at": "2025-12-20T10:30:00Z"
},
...
],
"total_size_bytes": 5368709120,
"count": 15
}
# Delete a torrent (manual cleanup)
DELETE /torrents/{torrent_id}
Query params: ?delete_data=true # Also delete downloaded files
Response:
{ "status": "deleted" }
# Trigger cleanup (free disk space)
POST /torrents/cleanup
Request:
{
"strategy": "oldest", # oldest, largest, lowest_ratio
"target_free_gb": 10 # Keep removing until this much free
}
Response:
{
"removed_count": 3,
"freed_bytes": 1073741824,
"free_space_gb": 10.5
}
# Health check
GET /health
Response:
{
"status": "healthy",
"transmission": {
"available": true,
"version": "4.0.5",
"active_torrents": 5
},
"disk": {
"total_gb": 30,
"used_gb": 18,
"free_gb": 12
},
"providers": {
"redacted": true,
"ops": true,
"youtube": true
}
}
```
**Torrent Downloader Changes:**
```python
# flacfetch/downloaders/torrent.py
class TorrentDownloader(Downloader):
def __init__(self, ..., keep_seeding: bool = False):
"""
Args:
keep_seeding: If True, don't remove torrent after download.
Used in server mode for indefinite seeding.
"""
self.keep_seeding = keep_seeding
def download(self, release, output_path, output_filename=None):
# ... existing download logic ...
# After download complete:
if self.keep_seeding:
# Don't remove, just return the file path
logger.info(f"Download complete, keeping torrent for seeding: {torrent.name}")
else:
# Original behavior: stop and remove
self.client.stop_torrent(torrent.id)
self.client.remove_torrent(torrent.id, delete_data=True)
```
**CLI Changes:**
```bash
# New serve subcommand
flacfetch serve --port 8080 --host 0.0.0.0
# Or via environment
FLACFETCH_API_PORT=8080 flacfetch serve
```
### Phase 2: Infrastructure (Pulumi)
**Location:** `./infrastructure/__main__.py`
**New Resources:**
```python
# Static IP for flacfetch service (for tracker whitelist)
flacfetch_ip = gcp.compute.Address(
"flacfetch-ip",
name="flacfetch-static-ip",
region="us-central1",
)
# Service account for flacfetch VM
flacfetch_sa = gcp.serviceaccount.Account(
"flacfetch-sa",
account_id="flacfetch-service",
display_name="Flacfetch Service Account",
)
# GCS write permissions for flacfetch
gcp.storage.BucketIAMMember(
"flacfetch-gcs-writer",
bucket=bucket.name,
role="roles/storage.objectCreator",
member=pulumi.Output.concat("serviceAccount:", flacfetch_sa.email),
)
# Flacfetch VM
flacfetch_vm = gcp.compute.Instance(
"flacfetch-service",
machine_type="e2-small",
zone="us-central1-a",
boot_disk=gcp.compute.InstanceBootDiskArgs(
initialize_params=gcp.compute.InstanceBootDiskInitializeParamsArgs(
image="debian-cloud/debian-12",
size=30, # GB
type="pd-ssd",
),
),
network_interfaces=[gcp.compute.InstanceNetworkInterfaceArgs(
network="default",
access_configs=[gcp.compute.InstanceNetworkInterfaceAccessConfigArgs(
nat_ip=flacfetch_ip.address, # Static IP
)],
)],
service_account=gcp.compute.InstanceServiceAccountArgs(
email=flacfetch_sa.email,
scopes=["cloud-platform"],
),
metadata_startup_script=FLACFETCH_STARTUP_SCRIPT,
tags=["flacfetch-service"],
)
# Firewall for flacfetch
flacfetch_firewall = gcp.compute.Firewall(
"flacfetch-firewall",
network="default",
allows=[
# BitTorrent peer connections
gcp.compute.FirewallAllowArgs(protocol="tcp", ports=["51413"]),
gcp.compute.FirewallAllowArgs(protocol="udp", ports=["51413"]),
# Flacfetch HTTP API (auth required at app level)
gcp.compute.FirewallAllowArgs(protocol="tcp", ports=["8080"]),
],
source_ranges=["0.0.0.0/0"], # Permissive, auth at app level
target_tags=["flacfetch-service"],
)
# Store API key in Secret Manager
flacfetch_api_key = gcp.secretmanager.Secret(
"flacfetch-api-key",
secret_id="flacfetch-api-key",
replication=gcp.secretmanager.SecretReplicationArgs(
auto=gcp.secretmanager.SecretReplicationAutoArgs(),
),
)
# Export for use
pulumi.export("flacfetch_static_ip", flacfetch_ip.address)
pulumi.export("flacfetch_service_url", pulumi.Output.concat("http://", flacfetch_ip.address, ":8080"))
```
**Startup Script:**
```bash
#!/bin/bash
set -e
# Install dependencies
apt-get update
apt-get install -y python3-pip python3-venv transmission-daemon ffmpeg git
# Configure Transmission
systemctl stop transmission-daemon
cat > /etc/transmission-daemon/settings.json << 'EOF'
{
"download-dir": "/var/lib/transmission-daemon/downloads",
"incomplete-dir": "/var/lib/transmission-daemon/.incomplete",
"incomplete-dir-enabled": true,
"rpc-authentication-required": false,
"rpc-bind-address": "127.0.0.1",
"rpc-enabled": true,
"rpc-port": 9091,
"rpc-whitelist-enabled": false,
"peer-port": 51413,
"port-forwarding-enabled": false,
"speed-limit-down": 0,
"speed-limit-down-enabled": false,
"speed-limit-up": 0,
"speed-limit-up-enabled": false,
"ratio-limit-enabled": false,
"umask": 2
}
EOF
systemctl start transmission-daemon
# Install flacfetch
cd /opt
git clone https://github.com/nomadkaraoke/flacfetch.git
cd flacfetch
python3 -m venv venv
source venv/bin/activate
pip install -e ".[api]"
# Get secrets from metadata/Secret Manager
FLACFETCH_API_KEY=$(gcloud secrets versions access latest --secret=flacfetch-api-key)
REDACTED_API_KEY=$(gcloud secrets versions access latest --secret=redacted-api-key)
OPS_API_KEY=$(gcloud secrets versions access latest --secret=ops-api-key)
GCS_BUCKET=$(curl -s "http://metadata.google.internal/computeMetadata/v1/instance/attributes/gcs-bucket" -H "Metadata-Flavor: Google")
# Create systemd service
cat > /etc/systemd/system/flacfetch.service << EOF
[Unit]
Description=Flacfetch HTTP API Service
After=network.target transmission-daemon.service
[Service]
Type=simple
User=root
WorkingDirectory=/opt/flacfetch
Environment="FLACFETCH_API_KEY=${FLACFETCH_API_KEY}"
Environment="REDACTED_API_KEY=${REDACTED_API_KEY}"
Environment="OPS_API_KEY=${OPS_API_KEY}"
Environment="GCS_BUCKET=${GCS_BUCKET}"
Environment="FLACFETCH_KEEP_SEEDING=true"
Environment="FLACFETCH_MIN_FREE_GB=5"
ExecStart=/opt/flacfetch/venv/bin/flacfetch serve --host 0.0.0.0 --port 8080
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
EOF
systemctl daemon-reload
systemctl enable flacfetch
systemctl start flacfetch
```
### Phase 3: Backend Integration
**New File:** `backend/services/flacfetch_client.py`
```python
"""
Client for the remote flacfetch HTTP API service.
Used when FLACFETCH_API_URL is configured, otherwise falls back to local flacfetch.
"""
import asyncio
import logging
from typing import Optional, List, Dict, Any
import httpx
logger = logging.getLogger(__name__)
class FlacfetchServiceError(Exception):
"""Error communicating with flacfetch service."""
pass
class FlacfetchClient:
"""Client for remote flacfetch HTTP API."""
def __init__(self, base_url: str, api_key: str, timeout: int = 30):
self.base_url = base_url.rstrip('/')
self.api_key = api_key
self.timeout = timeout
def _headers(self) -> Dict[str, str]:
return {"X-API-Key": self.api_key}
async def health_check(self) -> Dict[str, Any]:
"""Check if service is healthy."""
async with httpx.AsyncClient() as client:
resp = await client.get(
f"{self.base_url}/health",
headers=self._headers(),
timeout=10,
)
resp.raise_for_status()
return resp.json()
async def search(self, artist: str, title: str) -> Dict[str, Any]:
"""Search for audio."""
async with httpx.AsyncClient() as client:
resp = await client.post(
f"{self.base_url}/search",
headers=self._headers(),
json={"artist": artist, "title": title},
timeout=self.timeout,
)
resp.raise_for_status()
return resp.json()
async def download(
self,
search_id: str,
result_index: int,
output_filename: Optional[str] = None,
gcs_path: Optional[str] = None,
) -> str:
"""
Start download and return download_id.
If gcs_path is provided, file will be uploaded to GCS.
"""
async with httpx.AsyncClient() as client:
payload = {
"search_id": search_id,
"result_index": result_index,
}
if output_filename:
payload["output_filename"] = output_filename
if gcs_path:
payload["upload_to_gcs"] = True
payload["gcs_path"] = gcs_path
resp = await client.post(
f"{self.base_url}/download",
headers=self._headers(),
json=payload,
timeout=self.timeout,
)
resp.raise_for_status()
return resp.json()["download_id"]
async def get_download_status(self, download_id: str) -> Dict[str, Any]:
"""Get download status."""
async with httpx.AsyncClient() as client:
resp = await client.get(
f"{self.base_url}/download/{download_id}/status",
headers=self._headers(),
timeout=10,
)
resp.raise_for_status()
return resp.json()
async def wait_for_download(
self,
download_id: str,
timeout: int = 600,
poll_interval: int = 2,
) -> Dict[str, Any]:
"""
Wait for download to complete.
Returns final status dict with output_path or gcs_path.
Raises FlacfetchServiceError on failure or timeout.
"""
elapsed = 0
while elapsed < timeout:
status = await self.get_download_status(download_id)
if status["status"] == "complete":
return status
elif status["status"] == "failed":
raise FlacfetchServiceError(f"Download failed: {status.get('error')}")
await asyncio.sleep(poll_interval)
elapsed += poll_interval
raise FlacfetchServiceError(f"Download timed out after {timeout}s")
# Singleton
_client: Optional[FlacfetchClient] = None
def get_flacfetch_client() -> Optional[FlacfetchClient]:
"""Get flacfetch client if configured."""
global _client
if _client is None:
from backend.config import get_settings
settings = get_settings()
if settings.flacfetch_api_url and settings.flacfetch_api_key:
_client = FlacfetchClient(
base_url=settings.flacfetch_api_url,
api_key=settings.flacfetch_api_key,
)
return _client
```
**Modify:** `backend/services/audio_search_service.py` to use remote client when available.
**Modify:** `backend/config.py` to add new settings:
```python
flacfetch_api_url: Optional[str] = None # e.g., http://10.0.0.5:8080
flacfetch_api_key: Optional[str] = None
```
### Phase 4: Monitoring & Reliability
**Tasks:**
- [ ] Add Cloud Monitoring alert for VM CPU/memory
- [ ] Add alert for disk space < 5GB
- [ ] Add alert for flacfetch service down (health check fails)
- [ ] Configure Cloud Logging export from VM
- [ ] Add Grafana dashboard (optional)
**Automatic Disk Cleanup:**
The disk manager runs as a background task in the flacfetch API service:
```python
# Check every 5 minutes, cleanup if free space < threshold
async def background_cleanup_task():
while True:
await asyncio.sleep(300)
free_gb = get_free_space_gb()
if free_gb < MIN_FREE_GB:
cleanup_oldest_torrents(target_free_gb=MIN_FREE_GB + 5)
```
### Phase 5: Testing & Documentation
**Testing:**
- [ ] Unit tests for flacfetch API endpoints
- [ ] Unit tests for disk manager
- [ ] Integration test: search → download → GCS upload
- [ ] Mock tests for karaoke-gen backend client
**Documentation:**
- [ ] Update `docs/01-reference/ARCHITECTURE.md` with flacfetch service
- [ ] Add `flacfetch/README.md` HTTP API section
- [ ] Document deployment process
- [ ] Document environment variables
## Environment Variables
### Flacfetch Service (VM)
```bash
# API Configuration
FLACFETCH_API_KEY=<secret> # Required for API auth
FLACFETCH_API_PORT=8080 # Default: 8080
FLACFETCH_API_HOST=0.0.0.0 # Default: 0.0.0.0
# Tracker API Keys
REDACTED_API_KEY=<key> # For Redacted provider
OPS_API_KEY=<key> # For OPS provider
# GCS Upload (optional)
GCS_BUCKET=karaoke-uploads # For uploading completed downloads
GOOGLE_CLOUD_PROJECT=nomadkaraoke
# Transmission
TRANSMISSION_HOST=localhost # Default: localhost
TRANSMISSION_PORT=9091 # Default: 9091
# Behavior
FLACFETCH_KEEP_SEEDING=true # Keep torrents seeding after download
FLACFETCH_MIN_FREE_GB=5 # Minimum free disk space before cleanup
FLACFETCH_DOWNLOAD_DIR=/var/lib/transmission-daemon/downloads
```
### Cloud Run Backend
```bash
# New variables
FLACFETCH_API_URL=http://10.128.0.5:8080 # Internal IP of flacfetch VM
FLACFETCH_API_KEY=<same-secret>
```
## Security Considerations
1. **API Authentication**: Static API key in `X-API-Key` header
2. **Firewall**: Permissive for future web UI, auth enforced at application level
3. **GCS**: Service account with `storage.objectCreator` role only
4. **Transmission**: RPC bound to localhost only, no external access
5. **Static IP**: Can be whitelisted on tracker accounts
## Rollback Plan
If flacfetch service fails:
1. Set `FLACFETCH_API_URL=""` in Cloud Run (disables remote service)
2. Backend falls back to local flacfetch (YouTube only in Cloud Run)
3. Users see warning that lossless downloads may be slower/unavailable
## Cost Estimate
| Resource | Monthly Cost |
|----------|--------------|
| e2-small VM | ~$12 |
| Static IP | ~$3 |
| 30GB SSD | ~$5 |
| Network egress | ~$2-5 |
| **Total** | **~$22-25/month** |
## Success Criteria
- [ ] Flacfetch HTTP API accepts search requests and returns results
- [ ] Downloads complete successfully for Redacted/OPS sources
- [ ] Downloaded files upload to GCS automatically
- [ ] Torrents continue seeding after download
- [ ] Disk cleanup triggers automatically when space is low
- [ ] Remote CLI shows download progress from flacfetch service
- [ ] Fallback to local flacfetch works when service unavailable
- [ ] Static IP can be whitelisted on tracker accounts
## Timeline
| Phase | Duration | Dependencies |
|-------|----------|--------------|
| Phase 1: Flacfetch HTTP API | 1.5 days | None |
| Phase 2: Infrastructure | 0.5 day | Phase 1 |
| Phase 3: Backend Integration | 0.5 day | Phase 1, 2 |
| Phase 4: Monitoring | 0.5 day | Phase 2 |
| Phase 5: Testing & Docs | 0.5 day | Phase 1-3 |
| **Total** | **3.5 days** | |
## References
- [Transmission RPC Spec](https://github.com/transmission/transmission/blob/main/docs/rpc-spec.md)
- [transmission-rpc Python library](https://github.com/Trim21/transmission-rpc)
- [FastAPI Documentation](https://fastapi.tiangolo.com/)
- [flacfetch source code](./flacfetch/)
Submodule flacfetch deleted from 35ae8994b8.
-8
View File
@@ -94,14 +94,6 @@ echo -n "your-genius-key" | \
# Add Audio Separator API URL
echo -n "https://your-modal-url" | \
gcloud secrets versions add audio-separator-api-url --data-file=-
# Add Redacted API key (for flacfetch audio search - private tracker)
echo -n "your-redacted-api-key" | \
gcloud secrets versions add redacted-api-key --data-file=-
# Add OPS API key (for flacfetch audio search - private tracker)
echo -n "your-ops-api-key" | \
gcloud secrets versions add ops-api-key --data-file=-
```
Or use Pulumi config secrets:
-259
View File
@@ -428,24 +428,6 @@ audio_separator_secret = secretmanager.Secret(
),
)
# Redacted API Key Secret (for flacfetch audio search)
redacted_api_key_secret = secretmanager.Secret(
"redacted-api-key",
secret_id="redacted-api-key",
replication=secretmanager.SecretReplicationArgs(
auto=secretmanager.SecretReplicationAutoArgs(),
),
)
# OPS API Key Secret (for flacfetch audio search)
ops_api_key_secret = secretmanager.Secret(
"ops-api-key",
secret_id="ops-api-key",
replication=secretmanager.SecretReplicationArgs(
auto=secretmanager.SecretReplicationAutoArgs(),
),
)
# Create Cloud Run Domain Mapping
# Maps api.nomadkaraoke.com to the karaoke-backend service
domain_mapping = cloudrun.DomainMapping(
@@ -497,244 +479,3 @@ domain_mapping.statuses.apply(
})
)
# ==================== Flacfetch Service (Torrent Downloads) ====================
# A dedicated VM running flacfetch with Transmission daemon for BitTorrent downloads.
# This is required because Cloud Run doesn't support inbound peer connections for torrents.
# The VM has a static IP for tracker whitelist and runs indefinitely for seeding.
# Static IP for flacfetch (for tracker account whitelist)
flacfetch_ip = gcp.compute.Address(
"flacfetch-ip",
name="flacfetch-static-ip",
region="us-central1",
address_type="EXTERNAL",
description="Static IP for flacfetch service (torrent downloads)",
)
# Service account for flacfetch VM
flacfetch_sa = serviceaccount.Account(
"flacfetch-sa",
account_id="flacfetch-service",
display_name="Flacfetch Service Account",
description="Service account for flacfetch torrent/audio download VM",
)
# Grant flacfetch service account GCS write permissions
flacfetch_storage_iam = gcp.storage.BucketIAMMember(
"flacfetch-storage-writer",
bucket=bucket.name,
role="roles/storage.objectCreator",
member=flacfetch_sa.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant flacfetch service account GCS read permissions (for uploads folder)
flacfetch_storage_reader_iam = gcp.storage.BucketIAMMember(
"flacfetch-storage-reader",
bucket=bucket.name,
role="roles/storage.objectViewer",
member=flacfetch_sa.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Grant flacfetch service account Secret Manager access
flacfetch_secrets_iam = gcp.projects.IAMMember(
"flacfetch-secrets-access",
project=project_id,
role="roles/secretmanager.secretAccessor",
member=flacfetch_sa.email.apply(lambda email: f"serviceAccount:{email}"),
)
# Flacfetch API key secret (for authenticating API requests)
flacfetch_api_key_secret = secretmanager.Secret(
"flacfetch-api-key",
secret_id="flacfetch-api-key",
replication=secretmanager.SecretReplicationArgs(
auto=secretmanager.SecretReplicationAutoArgs(),
),
)
# Flacfetch API URL secret (the static IP address of the flacfetch VM)
# This is stored as a secret to avoid committing infrastructure IPs to the repo
flacfetch_api_url_secret = secretmanager.Secret(
"flacfetch-api-url",
secret_id="flacfetch-api-url",
replication=secretmanager.SecretReplicationArgs(
auto=secretmanager.SecretReplicationAutoArgs(),
),
)
# Startup script for the flacfetch VM
# This installs and configures Transmission daemon and the flacfetch service
FLACFETCH_STARTUP_SCRIPT = """#!/bin/bash
set -e
# Log to a file for debugging
exec > >(tee /var/log/flacfetch-startup.log) 2>&1
echo "Starting flacfetch setup at $(date)"
# Install dependencies
apt-get update
apt-get install -y python3-pip python3-venv transmission-daemon ffmpeg git curl
# Configure Transmission
echo "Configuring Transmission daemon..."
systemctl stop transmission-daemon || true
# Create transmission directories
mkdir -p /var/lib/transmission-daemon/downloads
mkdir -p /var/lib/transmission-daemon/.config/transmission-daemon
chown -R debian-transmission:debian-transmission /var/lib/transmission-daemon
# Configure transmission settings
cat > /var/lib/transmission-daemon/.config/transmission-daemon/settings.json << 'SETTINGS'
{
"download-dir": "/var/lib/transmission-daemon/downloads",
"incomplete-dir": "/var/lib/transmission-daemon/.incomplete",
"incomplete-dir-enabled": true,
"rpc-authentication-required": false,
"rpc-bind-address": "127.0.0.1",
"rpc-enabled": true,
"rpc-port": 9091,
"rpc-whitelist-enabled": false,
"peer-port": 51413,
"port-forwarding-enabled": false,
"speed-limit-down": 0,
"speed-limit-down-enabled": false,
"speed-limit-up": 0,
"speed-limit-up-enabled": false,
"ratio-limit-enabled": false,
"umask": 2,
"encryption": 1
}
SETTINGS
chown debian-transmission:debian-transmission /var/lib/transmission-daemon/.config/transmission-daemon/settings.json
systemctl start transmission-daemon
echo "Transmission daemon started"
# Install flacfetch
echo "Installing flacfetch..."
cd /opt
# Clone flacfetch (we'll use the repo from GitHub)
if [ ! -d "flacfetch" ]; then
git clone https://github.com/nomadkaraoke/flacfetch.git
fi
cd flacfetch
# Create and activate virtual environment
python3 -m venv venv
source venv/bin/activate
# Install flacfetch with API dependencies
pip install -e ".[api]"
# Get secrets from Secret Manager
echo "Fetching secrets from Secret Manager..."
FLACFETCH_API_KEY=$(gcloud secrets versions access latest --secret=flacfetch-api-key 2>/dev/null || echo "")
REDACTED_API_KEY=$(gcloud secrets versions access latest --secret=redacted-api-key 2>/dev/null || echo "")
OPS_API_KEY=$(gcloud secrets versions access latest --secret=ops-api-key 2>/dev/null || echo "")
# Get bucket name from project metadata
GCS_BUCKET=$(gcloud compute project-info describe --format='value(commonInstanceMetadata.items.gcs-bucket)' 2>/dev/null || echo "")
if [ -z "$GCS_BUCKET" ]; then
# Fallback: construct from project ID
PROJECT_ID=$(curl -s "http://metadata.google.internal/computeMetadata/v1/project/project-id" -H "Metadata-Flavor: Google")
GCS_BUCKET="karaoke-gen-storage-${PROJECT_ID}"
fi
echo "Using GCS bucket: $GCS_BUCKET"
# Create systemd service for flacfetch
echo "Creating systemd service..."
cat > /etc/systemd/system/flacfetch.service << SYSTEMD
[Unit]
Description=Flacfetch HTTP API Service
After=network.target transmission-daemon.service
Requires=transmission-daemon.service
[Service]
Type=simple
User=root
WorkingDirectory=/opt/flacfetch
Environment="FLACFETCH_API_KEY=${FLACFETCH_API_KEY}"
Environment="REDACTED_API_KEY=${REDACTED_API_KEY}"
Environment="OPS_API_KEY=${OPS_API_KEY}"
Environment="GCS_BUCKET=${GCS_BUCKET}"
Environment="FLACFETCH_KEEP_SEEDING=true"
Environment="FLACFETCH_MIN_FREE_GB=5"
Environment="FLACFETCH_DOWNLOAD_DIR=/var/lib/transmission-daemon/downloads"
Environment="TRANSMISSION_HOST=localhost"
Environment="TRANSMISSION_PORT=9091"
ExecStart=/opt/flacfetch/venv/bin/flacfetch serve --host 0.0.0.0 --port 8080
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target
SYSTEMD
systemctl daemon-reload
systemctl enable flacfetch
systemctl start flacfetch
echo "Flacfetch service started at $(date)"
echo "Setup complete!"
"""
# Flacfetch VM instance
flacfetch_vm = gcp.compute.Instance(
"flacfetch-service",
name="flacfetch-service",
machine_type="e2-small", # 0.5 vCPU, 2GB RAM - sufficient for I/O-bound workload
zone="us-central1-a",
boot_disk=gcp.compute.InstanceBootDiskArgs(
initialize_params=gcp.compute.InstanceBootDiskInitializeParamsArgs(
image="debian-cloud/debian-12",
size=30, # GB - for torrent storage + headroom
type="pd-ssd", # SSD for faster I/O
),
),
network_interfaces=[gcp.compute.InstanceNetworkInterfaceArgs(
network="default",
access_configs=[gcp.compute.InstanceNetworkInterfaceAccessConfigArgs(
nat_ip=flacfetch_ip.address, # Static IP
)],
)],
service_account=gcp.compute.InstanceServiceAccountArgs(
email=flacfetch_sa.email,
scopes=["cloud-platform"],
),
metadata_startup_script=FLACFETCH_STARTUP_SCRIPT,
metadata={
"gcs-bucket": bucket.name, # Pass bucket name to startup script
},
tags=["flacfetch-service"],
allow_stopping_for_update=True,
)
# Firewall rule for flacfetch
# - Port 51413 TCP/UDP: BitTorrent peer connections
# - Port 8080 TCP: Flacfetch HTTP API (auth required at app level)
flacfetch_firewall = gcp.compute.Firewall(
"flacfetch-firewall",
name="flacfetch-firewall",
network="default",
allows=[
# BitTorrent peer connections
gcp.compute.FirewallAllowArgs(protocol="tcp", ports=["51413"]),
gcp.compute.FirewallAllowArgs(protocol="udp", ports=["51413"]),
# Flacfetch HTTP API (auth required at app level)
gcp.compute.FirewallAllowArgs(protocol="tcp", ports=["8080"]),
],
source_ranges=["0.0.0.0/0"], # Permissive, auth enforced at app level
target_tags=["flacfetch-service"],
description="Firewall rules for flacfetch torrent/API service",
)
# Export flacfetch values
pulumi.export("flacfetch_static_ip", flacfetch_ip.address)
pulumi.export("flacfetch_service_url", flacfetch_ip.address.apply(
lambda ip: f"http://{ip}:8080"
))
pulumi.export("flacfetch_service_account", flacfetch_sa.email)
+1 -32
View File
@@ -4,35 +4,4 @@ import warnings
warnings.filterwarnings("ignore", category=SyntaxWarning, module="pydub.*")
warnings.filterwarnings("ignore", category=SyntaxWarning, module="syrics.*")
# Lazy imports to avoid loading heavy dependencies when not needed.
# KaraokePrep has many dependencies (lyrics_transcriber, etc.) which may not
# be available in all contexts (e.g., backend which only needs audio_fetcher).
#
# Use explicit imports instead:
# from karaoke_gen.karaoke_gen import KaraokePrep
# from karaoke_gen.audio_fetcher import FlacFetcher, AudioSearchResult, ...
__all__ = [
"KaraokePrep",
# Audio fetcher
"FlacFetcher",
"AudioSearchResult",
"AudioFetchResult",
"AudioFetcherError",
"NoResultsError",
"DownloadError",
"UserCancelledError",
]
def __getattr__(name):
"""Lazy import for heavy modules."""
if name == "KaraokePrep":
from .karaoke_gen import KaraokePrep
return KaraokePrep
elif name in ("FlacFetcher", "AudioSearchResult", "AudioFetchResult",
"AudioFetcherError", "NoResultsError", "DownloadError",
"UserCancelledError"):
from . import audio_fetcher
return getattr(audio_fetcher, name)
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
from .karaoke_gen import KaraokePrep
+53 -313
View File
@@ -13,8 +13,8 @@ import tempfile
import threading
from abc import ABC, abstractmethod
from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeoutError
from dataclasses import dataclass, asdict, field
from typing import List, Optional, Dict, Any
from dataclasses import dataclass
from typing import List, Optional
# Global flag to track if user requested cancellation via Ctrl+C
_interrupt_requested = False
@@ -22,14 +22,7 @@ _interrupt_requested = False
@dataclass
class AudioSearchResult:
"""Represents a single search result for audio.
Used by both local CLI and cloud backend. Supports serialization
for Firestore storage via to_dict()/from_dict().
For rich display, this class can serialize the full flacfetch Release
data so remote CLIs can use flacfetch's shared display functions.
"""
"""Represents a single search result for audio."""
title: str
artist: str
@@ -38,75 +31,13 @@ class AudioSearchResult:
duration: Optional[int] = None # Duration in seconds
quality: Optional[str] = None # e.g., "FLAC", "320kbps", etc.
source_id: Optional[str] = None # Unique ID from the source
index: int = 0 # Index in the results list (for API selection)
seeders: Optional[int] = None # Number of seeders (for torrent sources)
target_file: Optional[str] = None # Target filename in the release
# Raw result object from the provider (for download) - not serialized
raw_result: Optional[object] = field(default=None, repr=False)
def to_dict(self) -> Dict[str, Any]:
"""Convert to dict for JSON/Firestore serialization.
Includes full flacfetch Release data if available, enabling
remote CLIs to use flacfetch's shared display functions.
"""
result = {
"title": self.title,
"artist": self.artist,
"url": self.url,
"provider": self.provider,
"duration": self.duration,
"quality": self.quality,
"source_id": self.source_id,
"index": self.index,
"seeders": self.seeders,
"target_file": self.target_file,
}
# If we have a raw_result (flacfetch Release), include its full data
# This enables rich display on the remote CLI
if self.raw_result:
try:
release_dict = self.raw_result.to_dict()
# Merge Release fields into result (they may override basic fields)
for key in ['year', 'label', 'edition_info', 'release_type', 'channel',
'view_count', 'size_bytes', 'target_file_size', 'track_pattern',
'match_score', 'formatted_size', 'formatted_duration',
'formatted_views', 'is_lossless', 'quality_str', 'quality']:
if key in release_dict:
# Use 'quality_data' for the quality dict to avoid confusion with quality string
result_key = 'quality_data' if key == 'quality' else key
result[result_key] = release_dict[key]
except AttributeError:
pass # raw_result doesn't have to_dict() method
return result
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "AudioSearchResult":
"""Create from dict (e.g., from Firestore)."""
return cls(
title=data.get("title", ""),
artist=data.get("artist", ""),
url=data.get("url", ""),
provider=data.get("provider", "Unknown"),
duration=data.get("duration"),
quality=data.get("quality"),
source_id=data.get("source_id"),
index=data.get("index", 0),
seeders=data.get("seeders"),
target_file=data.get("target_file"),
raw_result=None, # Not stored in serialized form
)
# Raw result object from the provider (for download)
raw_result: Optional[object] = None
@dataclass
class AudioFetchResult:
"""Result of an audio fetch operation.
Used by both local CLI and cloud backend. Supports serialization
for Firestore storage via to_dict()/from_dict().
"""
"""Result of an audio fetch operation."""
filepath: str
artist: str
@@ -114,22 +45,6 @@ class AudioFetchResult:
provider: str
duration: Optional[int] = None
quality: Optional[str] = None
def to_dict(self) -> Dict[str, Any]:
"""Convert to dict for JSON/Firestore serialization."""
return asdict(self)
@classmethod
def from_dict(cls, data: Dict[str, Any]) -> "AudioFetchResult":
"""Create from dict (e.g., from Firestore)."""
return cls(
filepath=data.get("filepath", ""),
artist=data.get("artist", ""),
title=data.get("title", ""),
provider=data.get("provider", "Unknown"),
duration=data.get("duration"),
quality=data.get("quality"),
)
class AudioFetcherError(Exception):
@@ -245,8 +160,6 @@ class FlacFetchAudioFetcher(AudioFetcher):
This provides access to multiple audio sources including private music trackers
and YouTube, with intelligent prioritization of high-quality sources.
Also exported as FlacFetcher for shorter name.
"""
def __init__(
@@ -270,47 +183,6 @@ class FlacFetchAudioFetcher(AudioFetcher):
self._ops_api_key = ops_api_key or os.environ.get("OPS_API_KEY")
self._provider_priority = provider_priority
self._manager = None
self._transmission_available = None # Cached result of Transmission check
def _check_transmission_available(self) -> bool:
"""
Check if Transmission daemon is available for torrent downloads.
This prevents adding tracker providers (Redacted/OPS) when Transmission
isn't running, which would result in search results that can't be downloaded.
Returns:
True if Transmission is available and responsive, False otherwise.
"""
if self._transmission_available is not None:
self.logger.info(f"[Transmission] Using cached status: available={self._transmission_available}")
return self._transmission_available
host = os.environ.get("TRANSMISSION_HOST", "localhost")
port = int(os.environ.get("TRANSMISSION_PORT", "9091"))
self.logger.info(f"[Transmission] Checking availability at {host}:{port}")
try:
import transmission_rpc
self.logger.info(f"[Transmission] transmission_rpc imported successfully")
client = transmission_rpc.Client(host=host, port=port, timeout=5)
self.logger.info(f"[Transmission] Client created, calling session_stats()...")
# Simple test to verify connection works
stats = client.session_stats()
self.logger.info(f"[Transmission] Connected! Download dir: {getattr(stats, 'download_dir', 'unknown')}")
self._transmission_available = True
except ImportError as e:
self._transmission_available = False
self.logger.warning(f"[Transmission] transmission_rpc not installed: {e}")
except Exception as e:
self._transmission_available = False
self.logger.warning(f"[Transmission] Connection failed to {host}:{port}: {type(e).__name__}: {e}")
self.logger.info(f"[Transmission] Final status: available={self._transmission_available}")
return self._transmission_available
def _get_manager(self):
"""Lazily initialize and return the FetchManager."""
@@ -321,46 +193,30 @@ class FlacFetchAudioFetcher(AudioFetcher):
from flacfetch.downloaders.youtube import YoutubeDownloader
# Try to import TorrentDownloader (has optional dependencies)
TorrentDownloader = None
try:
from flacfetch.downloaders.torrent import TorrentDownloader
except ImportError:
TorrentDownloader = None
self.logger.debug("TorrentDownloader not available (missing dependencies)")
self._manager = FetchManager()
# Only add tracker providers if we can actually download from them
# This requires both TorrentDownloader and a running Transmission daemon
has_torrent_downloader = TorrentDownloader is not None
transmission_available = self._check_transmission_available()
can_use_trackers = has_torrent_downloader and transmission_available
self.logger.info(
f"[FlacFetcher] Provider setup: TorrentDownloader={has_torrent_downloader}, "
f"Transmission={transmission_available}, can_use_trackers={can_use_trackers}"
)
if not can_use_trackers and (self._redacted_api_key or self._ops_api_key):
self.logger.warning(
"[FlacFetcher] Tracker providers (Redacted/OPS) DISABLED: "
f"TorrentDownloader={has_torrent_downloader}, Transmission={transmission_available}. "
"Only YouTube sources will be used."
)
# Add providers and downloaders based on available API keys
if self._redacted_api_key and can_use_trackers:
if self._redacted_api_key:
from flacfetch.providers.redacted import RedactedProvider
self._manager.add_provider(RedactedProvider(api_key=self._redacted_api_key))
self._manager.register_downloader("Redacted", TorrentDownloader())
self.logger.info("[FlacFetcher] Added Redacted provider with TorrentDownloader")
if TorrentDownloader:
self._manager.register_downloader("Redacted", TorrentDownloader())
self.logger.debug("Added Redacted provider")
if self._ops_api_key and can_use_trackers:
if self._ops_api_key:
from flacfetch.providers.ops import OPSProvider
self._manager.add_provider(OPSProvider(api_key=self._ops_api_key))
self._manager.register_downloader("OPS", TorrentDownloader())
self.logger.info("[FlacFetcher] Added OPS provider with TorrentDownloader")
if TorrentDownloader:
self._manager.register_downloader("OPS", TorrentDownloader())
self.logger.debug("Added OPS provider")
# Always add YouTube as a fallback provider with its downloader
self._manager.add_provider(YoutubeProvider())
@@ -396,7 +252,7 @@ class FlacFetchAudioFetcher(AudioFetcher):
# Convert to our AudioSearchResult format
search_results = []
for i, result in enumerate(results):
for result in results:
# Get quality as string if it's a Quality object
quality = getattr(result, "quality", None)
quality_str = str(quality) if quality else None
@@ -410,9 +266,6 @@ class FlacFetchAudioFetcher(AudioFetcher):
duration=getattr(result, "duration_seconds", None),
quality=quality_str,
source_id=getattr(result, "info_hash", None),
index=i, # Set index for API selection
seeders=getattr(result, "seeders", None),
target_file=getattr(result, "target_file", None),
raw_result=result,
)
)
@@ -476,40 +329,6 @@ class FlacFetchAudioFetcher(AudioFetcher):
except Exception as e:
raise DownloadError(f"Failed to download {result.artist} - {result.title}: {e}") from e
def select_best(self, results: List[AudioSearchResult]) -> int:
"""
Select the best result from a list of search results.
Uses flacfetch's built-in quality ranking to determine the best source.
This is useful for automated/non-interactive usage.
Args:
results: List of AudioSearchResult objects from search()
Returns:
Index of the best result in the list
"""
if not results:
return 0
manager = self._get_manager()
# Get raw results that have raw_result set
raw_results = [r.raw_result for r in results if r.raw_result is not None]
if raw_results:
try:
best = manager.select_best(raw_results)
# Find index of best result
for i, r in enumerate(results):
if r.raw_result == best:
return i
except Exception as e:
self.logger.warning(f"select_best failed, using first result: {e}")
# Fallback: return first result
return 0
def search_and_download(
self,
artist: str,
@@ -577,15 +396,13 @@ class FlacFetchAudioFetcher(AudioFetcher):
self.logger.info(f"Downloading from {getattr(selected, 'source_name', 'Unknown')}...")
try:
# Use interruptible download so Ctrl+C works during torrent downloads
filepath = self._interruptible_download(
manager,
filepath = manager.download(
selected,
output_path=output_dir,
output_filename=output_filename,
)
if not filepath:
if filepath is None:
raise DownloadError(f"Download returned no file path for: {artist} - {title}")
self.logger.info(f"Downloaded to: {filepath}")
@@ -671,111 +488,6 @@ class FlacFetchAudioFetcher(AudioFetcher):
signal.signal(signal.SIGINT, original_handler)
_interrupt_requested = False
def _interruptible_download(self, manager, selected, output_path: str, output_filename: str) -> str:
"""
Run download in a way that can be interrupted by Ctrl+C.
The flacfetch/transmission download is a blocking operation that doesn't
respond to SIGINT while running (especially for torrent downloads).
This method runs it in a background thread and periodically checks for interrupts.
Args:
manager: The FetchManager instance
selected: The selected result to download
output_path: Directory to save the file
output_filename: Filename to save as
Returns:
Path to the downloaded file
Raises:
UserCancelledError: If user presses Ctrl+C during download
DownloadError: If download fails
"""
global _interrupt_requested
_interrupt_requested = False
result_container = {"filepath": None, "error": None}
was_cancelled = False
def do_download():
try:
result_container["filepath"] = manager.download(
selected,
output_path=output_path,
output_filename=output_filename,
)
except Exception as e:
result_container["error"] = e
# Set up signal handler for immediate response to Ctrl+C
original_handler = signal.getsignal(signal.SIGINT)
def interrupt_handler(signum, frame):
global _interrupt_requested
_interrupt_requested = True
# Print immediately so user knows it was received
print("\nCancelling download... please wait (may take a few seconds)", file=sys.stderr)
signal.signal(signal.SIGINT, interrupt_handler)
try:
# Start download in background thread
download_thread = threading.Thread(target=do_download, daemon=True)
download_thread.start()
# Wait for completion with periodic interrupt checks
while download_thread.is_alive():
download_thread.join(timeout=0.2) # Check every 200ms
if _interrupt_requested:
was_cancelled = True
# Clean up any pending torrents before raising
self._cleanup_transmission_torrents(selected)
raise UserCancelledError("Download cancelled by user (Ctrl+C)")
# Check for errors from the download
if result_container["error"] is not None:
raise result_container["error"]
return result_container["filepath"]
finally:
# Restore original signal handler
signal.signal(signal.SIGINT, original_handler)
_interrupt_requested = False
def _cleanup_transmission_torrents(self, selected) -> None:
"""
Clean up any torrents in Transmission that were started for this download.
Called when a download is cancelled to remove incomplete torrents and their data.
Args:
selected: The selected result that was being downloaded
"""
try:
import transmission_rpc
host = os.environ.get("TRANSMISSION_HOST", "localhost")
port = int(os.environ.get("TRANSMISSION_PORT", "9091"))
client = transmission_rpc.Client(host=host, port=port, timeout=5)
# Get the release name to match against torrents
release_name = getattr(selected, 'name', None) or getattr(selected, 'title', None)
if not release_name:
self.logger.debug("[Transmission] No release name to match for cleanup")
return
# Find and remove matching incomplete torrents
torrents = client.get_torrents()
for torrent in torrents:
# Match by name similarity and incomplete status
if torrent.progress < 100 and release_name.lower() in torrent.name.lower():
self.logger.info(f"[Transmission] Removing cancelled torrent: {torrent.name}")
client.remove_torrent(torrent.id, delete_data=True)
except Exception as e:
# Don't fail the cancellation if cleanup fails
self.logger.debug(f"[Transmission] Cleanup failed (non-fatal): {e}")
def _interactive_select(self, results: list, artist: str, title: str) -> object:
"""
Present search results to the user for interactive selection.
@@ -827,9 +539,41 @@ class FlacFetchAudioFetcher(AudioFetcher):
Raises:
UserCancelledError: If user cancels selection
"""
# Use flacfetch's shared display function
from flacfetch import print_releases
print_releases(results, target_artist=artist, use_colors=True)
print(f"\n{'=' * 60}")
print(f"Search Results for: {artist} - {title}")
print(f"{'=' * 60}\n")
for i, result in enumerate(results, 1):
# Use correct flacfetch attribute names
source_name = getattr(result, "source_name", "Unknown")
result_title = getattr(result, "title", "Unknown")
result_artist = getattr(result, "artist", "Unknown")
quality = getattr(result, "quality", None)
duration_seconds = getattr(result, "duration_seconds", None)
seeders = getattr(result, "seeders", None)
target_file = getattr(result, "target_file", None)
# Format duration if available
duration_str = ""
if duration_seconds:
minutes = duration_seconds // 60
seconds = duration_seconds % 60
duration_str = f" [{minutes}:{seconds:02d}]"
# Format quality - it's a Quality object with __str__
quality_str = f" ({quality})" if quality else ""
# Format seeders for torrent sources
seeders_str = f" Seeders: {seeders}" if seeders is not None else ""
# Format target file
file_str = f' "{target_file}"' if target_file else ""
print(f" {i}. [{source_name}] {result_artist} - {result_title}{quality_str}{duration_str}{seeders_str}{file_str}")
print()
print(" 0. Cancel")
print()
while True:
try:
@@ -854,10 +598,6 @@ class FlacFetchAudioFetcher(AudioFetcher):
raise UserCancelledError("Selection cancelled by user (Ctrl+C)")
# Alias for shorter name - used by backend and other consumers
FlacFetcher = FlacFetchAudioFetcher
def create_audio_fetcher(
logger: Optional[logging.Logger] = None,
redacted_api_key: Optional[str] = None,
+29 -112
View File
@@ -36,8 +36,6 @@ from typing import Any, Dict, Optional
import requests
from .cli_args import create_parser, process_style_overrides, is_url, is_file
# Use flacfetch's shared display functions for consistent formatting
from flacfetch import print_releases, Release
class JobStatus(str, Enum):
@@ -1331,54 +1329,6 @@ class JobMonitor:
"""Get user-friendly description for a status."""
return self.STATUS_DESCRIPTIONS.get(status, status)
def _show_download_progress(self, job_data: Dict[str, Any]) -> None:
"""Show detailed download progress during audio download."""
try:
# Get provider from job state_data
state_data = job_data.get('state_data', {})
provider = state_data.get('selected_audio_provider', 'unknown')
# For non-torrent providers (YouTube), just show simple message
if provider.lower() == 'youtube':
self.logger.info(f" [Downloading from YouTube...]")
return
# Query health endpoint for transmission status (torrent providers)
health_url = f"{self.config.service_url}/api/health/detailed"
response = requests.get(health_url, timeout=5)
if response.status_code == 200:
health = response.json()
transmission = health.get('dependencies', {}).get('transmission', {})
if transmission.get('available'):
torrents = transmission.get('torrents', [])
if torrents:
# Show info about active torrents
for t in torrents:
progress = t.get('progress', 0)
peers = t.get('peers', 0)
speed = t.get('download_speed', 0)
stalled = t.get('stalled', False)
if stalled:
self.logger.info(f" [Downloading from {provider}] {progress:.1f}% - STALLED (no peers)")
elif progress < 100:
self.logger.info(f" [Downloading from {provider}] {progress:.1f}% @ {speed:.1f} KB/s ({peers} peers)")
else:
self.logger.info(f" [Downloading from {provider}] Complete, processing...")
else:
# No torrents - might be starting or YouTube download
self.logger.info(f" [Downloading from {provider}] Starting download...")
else:
self.logger.info(f" [Downloading from {provider}] Transmission not available - download may fail")
else:
self.logger.info(f" [Downloading from {provider}]...")
except Exception as e:
# Fall back to simple message
self.logger.info(f" [Downloading audio...]")
def open_browser(self, url: str) -> None:
"""Open URL in the default browser."""
system = platform.system()
@@ -1567,47 +1517,6 @@ class JobMonitor:
except Exception as e:
self.logger.error(f"Error submitting selection: {e}")
def _convert_api_result_to_release_dict(self, result: dict) -> dict:
"""
Convert API search result to a dict compatible with flacfetch's Release.from_dict().
This enables using flacfetch's shared display functions for consistent,
rich formatting between local and remote CLIs.
"""
# Build quality dict from API response
quality_data = result.get('quality_data') or {
"format": "OTHER",
"media": "OTHER",
}
return {
"title": result.get('title', ''),
"artist": result.get('artist', ''),
"source_name": result.get('provider', 'Unknown'),
"download_url": result.get('url'),
"info_hash": result.get('source_id'),
"size_bytes": result.get('size_bytes'),
"year": result.get('year'),
"edition_info": result.get('edition_info'),
"label": result.get('label'),
"release_type": result.get('release_type'),
"seeders": result.get('seeders'),
"channel": result.get('channel'),
"view_count": result.get('view_count'),
"duration_seconds": result.get('duration'),
"target_file": result.get('target_file'),
"target_file_size": result.get('target_file_size'),
"track_pattern": result.get('track_pattern'),
"match_score": result.get('match_score', 0.0),
"quality": quality_data,
# Pre-computed fields
"formatted_size": result.get('formatted_size'),
"formatted_duration": result.get('formatted_duration'),
"formatted_views": result.get('formatted_views'),
"is_lossless": result.get('is_lossless', False),
"quality_str": result.get('quality_str') or result.get('quality', ''),
}
def handle_audio_selection(self, job_id: str) -> None:
"""Handle audio source selection interaction (Batch 5)."""
self.logger.info("=" * 60)
@@ -1618,7 +1527,6 @@ class JobMonitor:
# Get search results
results_data = self.client.get_audio_search_results(job_id)
results = results_data.get('results', [])
artist = results_data.get('artist', 'Unknown')
if not results:
self.logger.error("No search results available")
@@ -1629,27 +1537,42 @@ class JobMonitor:
self.logger.info("Non-interactive mode: Auto-selecting first result")
selection_index = 0
else:
# Convert API results to Release-compatible dicts for flacfetch display
# This gives us the same rich, colorized output as the local CLI
release_dicts = [self._convert_api_result_to_release_dict(r) for r in results]
self.logger.info("")
self.logger.info("Choose which audio source to download:")
self.logger.info("")
# Use flacfetch's shared display function
print_releases(release_dicts, target_artist=artist, use_colors=True)
for result in results:
index = result.get('index', 0)
provider = result.get('provider', 'Unknown')
artist = result.get('artist', 'Unknown')
title = result.get('title', 'Unknown')
quality = result.get('quality', '')
duration = result.get('duration')
# Format duration if available
duration_str = ""
if duration:
minutes = duration // 60
seconds = duration % 60
duration_str = f" [{minutes}:{seconds:02d}]"
quality_str = f" ({quality})" if quality else ""
self.logger.info(f" {index + 1}) [{provider}] {artist} - {title}{quality_str}{duration_str}")
self.logger.info("")
selection_index = -1
while selection_index < 0:
try:
choice = input(f"\nSelect a release (1-{len(results)}, 0 to cancel): ").strip()
if choice == "0":
self.logger.info("Selection cancelled by user")
raise KeyboardInterrupt
choice = input(f"Enter your choice (1-{len(results)}): ").strip()
choice_num = int(choice)
if 1 <= choice_num <= len(results):
selection_index = choice_num - 1
else:
print(f"Please enter a number between 0 and {len(results)}")
self.logger.error(f"Please enter a number between 1 and {len(results)}")
except ValueError:
print("Please enter a valid number")
self.logger.error("Please enter a valid number")
except KeyboardInterrupt:
print()
raise
@@ -2115,15 +2038,9 @@ class JobMonitor:
self._polls_without_updates = 0
else:
self._polls_without_updates += 1
# More frequent updates during audio download (every poll)
heartbeat_threshold = 1 if status == 'downloading_audio' else self._heartbeat_interval
if self._polls_without_updates >= heartbeat_threshold:
if status == 'downloading_audio':
# Show detailed download progress including transmission status
self._show_download_progress(job_data)
else:
description = self._get_status_description(status)
self.logger.info(f" [Still processing: {description}]")
if self._polls_without_updates >= self._heartbeat_interval:
description = self._get_status_description(status)
self.logger.info(f" [Still processing: {description}]")
self._polls_without_updates = 0
# Handle human interaction points
Generated
+407 -403
View File
File diff suppressed because it is too large. Load diff
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.73.3"
version = "0.72.0"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"
File diff suppressed because it is too large. Load diff
-198
View File
@@ -2038,201 +2038,3 @@ class TestSignedUrlUploadFlow:
assert hasattr(client, '_get_content_type')
assert callable(client._upload_file_to_signed_url)
assert callable(client._get_content_type)
class TestJobMonitorDownloadProgress:
"""Tests for JobMonitor download progress display."""
@pytest.fixture
def config(self):
return Config(
service_url="https://api.example.com",
review_ui_url="https://review.example.com",
poll_interval=1,
output_dir="/tmp/output"
)
@pytest.fixture
def logger(self):
return logging.getLogger("test")
@pytest.fixture
def monitor(self, config, logger):
client = MagicMock()
return JobMonitor(client, config, logger)
def test_show_download_progress_youtube(self, monitor, caplog):
"""Test that YouTube downloads show simple message."""
job_data = {
'state_data': {
'selected_audio_provider': 'YouTube'
}
}
with caplog.at_level(logging.INFO):
monitor._show_download_progress(job_data)
assert "Downloading from YouTube" in caplog.text
@patch('karaoke_gen.utils.remote_cli.requests.get')
def test_show_download_progress_torrent_with_active_torrents(self, mock_get, monitor, caplog):
"""Test that active torrents show progress details."""
job_data = {
'state_data': {
'selected_audio_provider': 'Redacted'
}
}
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
'dependencies': {
'transmission': {
'available': True,
'torrents': [{
'progress': 45.5,
'peers': 3,
'download_speed': 1250.0,
'stalled': False
}]
}
}
}
mock_get.return_value = mock_response
with caplog.at_level(logging.INFO):
monitor._show_download_progress(job_data)
assert "Downloading from Redacted" in caplog.text
assert "45.5%" in caplog.text
assert "1250.0 KB/s" in caplog.text
assert "3 peers" in caplog.text
@patch('karaoke_gen.utils.remote_cli.requests.get')
def test_show_download_progress_torrent_stalled(self, mock_get, monitor, caplog):
"""Test that stalled torrents show warning."""
job_data = {
'state_data': {
'selected_audio_provider': 'OPS'
}
}
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
'dependencies': {
'transmission': {
'available': True,
'torrents': [{
'progress': 0.0,
'peers': 0,
'download_speed': 0,
'stalled': True
}]
}
}
}
mock_get.return_value = mock_response
with caplog.at_level(logging.INFO):
monitor._show_download_progress(job_data)
assert "STALLED" in caplog.text
assert "no peers" in caplog.text
@patch('karaoke_gen.utils.remote_cli.requests.get')
def test_show_download_progress_no_torrents(self, mock_get, monitor, caplog):
"""Test message when no active torrents."""
job_data = {
'state_data': {
'selected_audio_provider': 'Redacted'
}
}
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
'dependencies': {
'transmission': {
'available': True,
'torrents': []
}
}
}
mock_get.return_value = mock_response
with caplog.at_level(logging.INFO):
monitor._show_download_progress(job_data)
assert "Starting download" in caplog.text
@patch('karaoke_gen.utils.remote_cli.requests.get')
def test_show_download_progress_transmission_unavailable(self, mock_get, monitor, caplog):
"""Test warning when Transmission not available."""
job_data = {
'state_data': {
'selected_audio_provider': 'Redacted'
}
}
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.json.return_value = {
'dependencies': {
'transmission': {
'available': False
}
}
}
mock_get.return_value = mock_response
with caplog.at_level(logging.INFO):
monitor._show_download_progress(job_data)
assert "not available" in caplog.text
@patch('karaoke_gen.utils.remote_cli.requests.get')
def test_show_download_progress_request_error(self, mock_get, monitor, caplog):
"""Test fallback message on request error."""
job_data = {
'state_data': {
'selected_audio_provider': 'Redacted'
}
}
mock_get.side_effect = Exception("Connection error")
with caplog.at_level(logging.INFO):
monitor._show_download_progress(job_data)
assert "Downloading audio" in caplog.text
@patch('karaoke_gen.utils.remote_cli.requests.get')
def test_show_download_progress_http_error(self, mock_get, monitor, caplog):
"""Test fallback on non-200 HTTP response."""
job_data = {
'state_data': {
'selected_audio_provider': 'Redacted'
}
}
mock_response = MagicMock()
mock_response.status_code = 500
mock_get.return_value = mock_response
with caplog.at_level(logging.INFO):
monitor._show_download_progress(job_data)
assert "Downloading from Redacted" in caplog.text
def test_show_download_progress_missing_provider(self, monitor, caplog):
"""Test handling of missing provider in state_data."""
job_data = {
'state_data': {}
}
with caplog.at_level(logging.INFO):
monitor._show_download_progress(job_data)
# Should use 'unknown' as default
assert "unknown" in caplog.text.lower() or "Downloading" in caplog.text