diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 96385d3a..a0da26c4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -902,6 +902,22 @@ jobs: # - / (main app) # - /app/jobs/{id}/lyrics (lyrics review) # - /app/jobs/{id}/instrumental (instrumental review) + - name: Capture build metadata + id: frontend-meta + run: | + echo "sha=${GITHUB_SHA::8}" >> $GITHUB_OUTPUT + PR_NUM=$(git log -1 --format=%s | grep -oP '\(#\K[0-9]+' | head -1 || echo "") + if [ -n "$PR_NUM" ]; then + PR_TITLE=$(gh pr view "$PR_NUM" --json title -q .title 2>/dev/null || echo "") + else + PR_NUM="" + PR_TITLE="" + fi + echo "pr_number=${PR_NUM}" >> $GITHUB_OUTPUT + echo "pr_title=${PR_TITLE}" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Install frontend dependencies working-directory: frontend run: npm ci --legacy-peer-deps @@ -911,6 +927,10 @@ jobs: working-directory: frontend env: NEXT_PUBLIC_API_URL: https://api.nomadkaraoke.com + NEXT_PUBLIC_COMMIT_SHA: ${{ steps.frontend-meta.outputs.sha }} + NEXT_PUBLIC_PR_NUMBER: ${{ steps.frontend-meta.outputs.pr_number }} + NEXT_PUBLIC_PR_TITLE: ${{ steps.frontend-meta.outputs.pr_title }} + NEXT_PUBLIC_BUILD_TIME: ${{ github.event.head_commit.timestamp }} run: npm run build - name: Deploy consumer to Cloudflare Pages @@ -929,6 +949,10 @@ jobs: env: NEXT_PUBLIC_API_URL: https://api.nomadkaraoke.com NEXT_PUBLIC_TENANT_PORTAL: "true" + NEXT_PUBLIC_COMMIT_SHA: ${{ steps.frontend-meta.outputs.sha }} + NEXT_PUBLIC_PR_NUMBER: ${{ steps.frontend-meta.outputs.pr_number }} + NEXT_PUBLIC_PR_TITLE: ${{ steps.frontend-meta.outputs.pr_title }} + NEXT_PUBLIC_BUILD_TIME: ${{ github.event.head_commit.timestamp }} run: npm run build - name: Deploy tenant portal to Cloudflare Pages @@ -1008,6 +1032,23 @@ jobs: echo "📋 Current version: $VERSION" echo "🚀 Deploying on every push to main" + - name: Capture build metadata + id: build-meta + run: | + echo "sha=${GITHUB_SHA::8}" >> $GITHUB_OUTPUT + # For squash-merge commits, PR number is in the commit message as (#NNN) + PR_NUM=$(git log -1 --format=%s | grep -oP '\(#\K[0-9]+' | head -1 || echo "") + if [ -n "$PR_NUM" ]; then + PR_TITLE=$(gh pr view "$PR_NUM" --json title -q .title 2>/dev/null || echo "") + else + PR_NUM="" + PR_TITLE="" + fi + echo "pr_number=${PR_NUM}" >> $GITHUB_OUTPUT + echo "pr_title=${PR_TITLE}" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Authenticate to Google Cloud (Workload Identity) id: auth uses: google-github-actions/auth@v2 @@ -1569,7 +1610,7 @@ jobs: --min-instances 1 \ --vpc-connector projects/nomadkaraoke/locations/us-central1/connectors/cloud-run-connector \ --vpc-egress private-ranges-only \ - --set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD,DEFAULT_PRIVATE_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-NonPublished,DEFAULT_PRIVATE_BRAND_PREFIX=NOMADNP,USE_GCE_ENCODING=true,USE_GCE_PREVIEW_ENCODING=true,ENABLE_PUSH_NOTIFICATIONS=true,USE_CLOUD_RUN_JOBS_FOR_VIDEO=true" \ + --set-env-vars "GOOGLE_CLOUD_PROJECT=${{ env.PROJECT_ID }},GCS_BUCKET_NAME=karaoke-gen-storage-${{ env.PROJECT_ID }},FIRESTORE_COLLECTION=jobs,ENVIRONMENT=production,KARAOKE_GEN_VERSION=$VERSION,DEFAULT_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-Organized,DEFAULT_GDRIVE_FOLDER_ID=1laRKAyxo0v817SstfM5XkpbWiNKNAMSX,ENABLE_CLOUD_TASKS=true,CLOUD_RUN_SERVICE_URL=https://api.nomadkaraoke.com,GCP_REGION=us-central1,DEFAULT_ENABLE_YOUTUBE_UPLOAD=true,DEFAULT_BRAND_PREFIX=NOMAD,DEFAULT_PRIVATE_DROPBOX_PATH=/MediaUnsynced/Karaoke/Tracks-NonPublished,DEFAULT_PRIVATE_BRAND_PREFIX=NOMADNP,USE_GCE_ENCODING=true,USE_GCE_PREVIEW_ENCODING=true,ENABLE_PUSH_NOTIFICATIONS=true,USE_CLOUD_RUN_JOBS_FOR_VIDEO=true,COMMIT_SHA=${{ steps.build-meta.outputs.sha }},PR_NUMBER=${{ steps.build-meta.outputs.pr_number }},PR_TITLE=${{ steps.build-meta.outputs.pr_title }}" \ --set-secrets "AUDIOSHAKE_API_TOKEN=audioshake-api-key:latest,GENIUS_API_TOKEN=genius-api-key:latest,SPOTIFY_COOKIE_SP_DC=spotify-cookie:latest,RAPIDAPI_KEY=rapidapi-key:latest,DEFAULT_DISCORD_WEBHOOK_URL=discord-releases-webhook:latest,ADMIN_TOKENS=admin-tokens:latest,RED_API_KEY=red-api-key:latest,RED_API_URL=red-api-url:latest,OPS_API_KEY=ops-api-key:latest,OPS_API_URL=ops-api-url:latest,FLACFETCH_API_URL=flacfetch-api-url:latest,FLACFETCH_API_KEY=flacfetch-api-key:latest,SENDGRID_API_KEY=sendgrid-api-key:latest,STRIPE_SECRET_KEY=stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=stripe-webhook-secret:latest,LANGFUSE_PUBLIC_KEY=langfuse-public-key:latest,LANGFUSE_SECRET_KEY=langfuse-secret-key:latest,LANGFUSE_HOST=langfuse-host:latest,ENCODING_WORKER_URL=encoding-worker-url:latest,ENCODING_WORKER_API_KEY=encoding-worker-api-key:latest,VAPID_PUBLIC_KEY=vapid-public-key:latest,VAPID_PRIVATE_KEY=vapid-private-key:latest,E2E_BYPASS_KEY=e2e-bypass-key:latest,GDRIVE_VALIDATOR_URL=gdrive-validator-url:latest" - name: Update Cloud Run Jobs with new image diff --git a/backend/api/routes/health.py b/backend/api/routes/health.py index 05dee551..4946cd35 100644 --- a/backend/api/routes/health.py +++ b/backend/api/routes/health.py @@ -5,10 +5,12 @@ import os import logging import subprocess import platform -from fastapi import APIRouter -from typing import Dict, Any, Optional +from fastapi import APIRouter, Depends +from typing import Dict, Any, Optional, Tuple -from backend.version import VERSION +from backend.version import VERSION, COMMIT_SHA, PR_NUMBER, PR_TITLE, STARTUP_TIME +from backend.api.dependencies import optional_auth +from backend.services.auth_service import UserType from backend.services.flacfetch_client import get_flacfetch_client from backend.services.email_service import get_email_service from backend.services.stripe_service import get_stripe_service @@ -308,6 +310,30 @@ async def flacfetch_health() -> Dict[str, Any]: } +def check_audio_separator_status() -> Dict[str, Any]: + """Check audio separator availability without calling the remote GPU service.""" + from backend.config import get_settings + + settings = get_settings() + if not settings.audio_separator_api_url: + return { + "available": False, + "status": "not_configured", + } + + try: + from importlib.metadata import version + pkg_version = version("audio-separator") + except Exception: + pkg_version = None + + return { + "available": True, + "status": "ok", + "version": pkg_version, + } + + @router.get("/health/audio-separator") async def audio_separator_health() -> Dict[str, Any]: """ @@ -341,6 +367,118 @@ async def audio_separator_health() -> Dict[str, Any]: } +def _get_encoding_worker_manager(): + """Get the EncodingWorkerManager instance. Separated for testability.""" + try: + encoding_service = get_encoding_service() + if hasattr(encoding_service, '_worker_manager') and encoding_service._worker_manager: + return encoding_service._worker_manager + except Exception: + pass + return None + + +@router.get("/health/system-status") +async def system_status( + auth: Optional[Tuple] = Depends(optional_auth), +) -> Dict[str, Any]: + """ + Aggregated system status for all services. + + Returns basic info for everyone. Admin users get additional + blue-green deployment details for the encoder. + """ + is_admin = auth is not None and len(auth) >= 2 and auth[1] == UserType.ADMIN + + # Fetch all health statuses concurrently + import asyncio + encoding_task = check_encoding_worker_status() + flacfetch_task = check_flacfetch_service_status() + encoding_status, flacfetch_status = await asyncio.gather( + encoding_task, flacfetch_task + ) + separator_status = check_audio_separator_status() + + # Build frontend service info (from env vars baked at build time — returned as-is) + frontend_svc = { + "status": "ok", + "version": VERSION, # Frontend and backend share the same version from pyproject.toml + } + + # Build backend service info + backend_svc = { + "status": "ok", + "version": VERSION, + "deployed_at": STARTUP_TIME, + } + if COMMIT_SHA: + backend_svc["commit_sha"] = COMMIT_SHA + if PR_NUMBER: + backend_svc["pr_number"] = PR_NUMBER + if PR_TITLE: + backend_svc["pr_title"] = PR_TITLE + + # Build encoder service info + encoder_svc = { + "status": "ok" if encoding_status.get("available") else "offline", + "version": encoding_status.get("wheel_version"), + "active_jobs": encoding_status.get("active_jobs", 0), + } + + # Build flacfetch service info + flacfetch_svc = { + "status": "ok" if flacfetch_status.get("available") else "offline", + "version": flacfetch_status.get("version"), + } + + # Build separator service info + separator_svc = { + "status": "ok" if separator_status.get("available") else "offline", + "version": separator_status.get("version"), + } + + # Admin-only: blue-green encoder details from Firestore + if is_admin: + manager = _get_encoding_worker_manager() + if manager: + try: + config = manager.get_config() + encoder_svc["admin_details"] = { + "primary_vm": config.primary_vm, + "primary_ip": config.primary_ip, + "primary_version": config.primary_version, + "primary_deployed_at": config.primary_deployed_at, + "secondary_vm": config.secondary_vm, + "secondary_ip": config.secondary_ip, + "secondary_version": config.secondary_version, + "secondary_deployed_at": config.secondary_deployed_at, + "last_swap_at": config.last_swap_at, + "deploy_in_progress": config.deploy_in_progress, + "active_jobs": encoding_status.get("active_jobs", 0), + "queue_length": encoding_status.get("queue_length", 0), + } + except Exception as e: + logger.warning(f"Failed to get blue-green config: {e}") + + # Add error details for offline services + if not flacfetch_status.get("available"): + flacfetch_svc["admin_details"] = {"error": flacfetch_status.get("error")} + if not separator_status.get("available"): + separator_svc["admin_details"] = {"error": separator_status.get("error")} + if not encoding_status.get("available") and "admin_details" not in encoder_svc: + encoder_svc["admin_details"] = {"error": encoding_status.get("error")} + + return { + "services": { + "frontend": frontend_svc, + "backend": backend_svc, + "encoder": encoder_svc, + "flacfetch": flacfetch_svc, + "separator": separator_svc, + } + } + + @router.get("/health/detailed") async def detailed_health_check() -> Dict[str, Any]: """ diff --git a/backend/tests/test_system_status_endpoint.py b/backend/tests/test_system_status_endpoint.py new file mode 100644 index 00000000..46fbaee1 --- /dev/null +++ b/backend/tests/test_system_status_endpoint.py @@ -0,0 +1,157 @@ +"""Tests for the /api/health/system-status endpoint.""" +import pytest +from unittest.mock import patch, MagicMock, AsyncMock + + +@pytest.fixture +def mock_encoding_status(): + return { + "configured": True, + "enabled": True, + "available": True, + "status": "ok", + "active_jobs": 0, + "queue_length": 0, + "wheel_version": "0.155.3", + } + + +@pytest.fixture +def mock_flacfetch_status(): + return { + "configured": True, + "available": True, + "status": "ok", + "version": "0.19.1", + } + + +@pytest.fixture +def mock_separator_status(): + return { + "available": True, + "status": "ok", + "version": "0.4.3", + } + + +@pytest.fixture +def mock_encoding_worker_config(): + """Mock EncodingWorkerConfig dataclass.""" + config = MagicMock() + config.primary_vm = "encoding-worker-b" + config.primary_ip = "34.10.189.118" + config.primary_version = "0.155.3" + config.primary_deployed_at = "2026-03-31T18:10:00Z" + config.secondary_vm = "encoding-worker-a" + config.secondary_ip = "34.57.78.246" + config.secondary_version = "0.155.2" + config.secondary_deployed_at = "2026-03-29T14:00:00Z" + config.last_swap_at = "2026-03-31T18:10:00Z" + config.deploy_in_progress = False + config.deploy_in_progress_since = None + config.last_activity_at = None + return config + + +class TestSystemStatusPublic: + """Tests for unauthenticated /api/health/system-status.""" + + @pytest.mark.asyncio + async def test_returns_all_services( + self, mock_encoding_status, mock_flacfetch_status, mock_separator_status + ): + from backend.api.routes.health import system_status + + with patch("backend.api.routes.health.check_encoding_worker_status", new_callable=AsyncMock, return_value=mock_encoding_status), \ + patch("backend.api.routes.health.check_flacfetch_service_status", new_callable=AsyncMock, return_value=mock_flacfetch_status), \ + patch("backend.api.routes.health.check_audio_separator_status", return_value=mock_separator_status), \ + patch("backend.api.routes.health.VERSION", "0.155.3"), \ + patch("backend.api.routes.health.COMMIT_SHA", "8e2a949a"), \ + patch("backend.api.routes.health.PR_NUMBER", "644"), \ + patch("backend.api.routes.health.PR_TITLE", "fix: blue-green deploy"), \ + patch("backend.api.routes.health.STARTUP_TIME", "2026-03-31T18:10:00Z"): + + result = await system_status(auth=None) + + assert "services" in result + services = result["services"] + + # All 5 services present + assert "frontend" in services + assert "backend" in services + assert "encoder" in services + assert "flacfetch" in services + assert "separator" in services + + # Backend has build metadata + backend_svc = services["backend"] + assert backend_svc["version"] == "0.155.3" + assert backend_svc["commit_sha"] == "8e2a949a" + assert backend_svc["pr_number"] == "644" + + # Encoder has status + assert services["encoder"]["status"] == "ok" + assert services["encoder"]["version"] == "0.155.3" + + # No admin_details without auth + assert "admin_details" not in services["encoder"] + + @pytest.mark.asyncio + async def test_offline_service(self, mock_flacfetch_status, mock_separator_status): + from backend.api.routes.health import system_status + + offline_encoding = { + "configured": True, + "enabled": True, + "available": False, + "error": "Connection refused", + } + + with patch("backend.api.routes.health.check_encoding_worker_status", new_callable=AsyncMock, return_value=offline_encoding), \ + patch("backend.api.routes.health.check_flacfetch_service_status", new_callable=AsyncMock, return_value=mock_flacfetch_status), \ + patch("backend.api.routes.health.check_audio_separator_status", return_value=mock_separator_status), \ + patch("backend.api.routes.health.VERSION", "0.155.3"), \ + patch("backend.api.routes.health.COMMIT_SHA", ""), \ + patch("backend.api.routes.health.PR_NUMBER", ""), \ + patch("backend.api.routes.health.PR_TITLE", ""), \ + patch("backend.api.routes.health.STARTUP_TIME", "2026-03-31T18:10:00Z"): + + result = await system_status(auth=None) + + assert result["services"]["encoder"]["status"] == "offline" + + +class TestSystemStatusAdmin: + """Tests for admin-authenticated /api/health/system-status.""" + + @pytest.mark.asyncio + async def test_admin_gets_bluegreen_details( + self, mock_encoding_status, mock_flacfetch_status, mock_separator_status, mock_encoding_worker_config + ): + from backend.api.routes.health import system_status + from backend.services.auth_service import UserType + + mock_auth = ("token123", UserType.ADMIN, -1) + mock_manager = MagicMock() + mock_manager.get_config.return_value = mock_encoding_worker_config + + with patch("backend.api.routes.health.check_encoding_worker_status", new_callable=AsyncMock, return_value=mock_encoding_status), \ + patch("backend.api.routes.health.check_flacfetch_service_status", new_callable=AsyncMock, return_value=mock_flacfetch_status), \ + patch("backend.api.routes.health.check_audio_separator_status", return_value=mock_separator_status), \ + patch("backend.api.routes.health.VERSION", "0.155.3"), \ + patch("backend.api.routes.health.COMMIT_SHA", ""), \ + patch("backend.api.routes.health.PR_NUMBER", ""), \ + patch("backend.api.routes.health.PR_TITLE", ""), \ + patch("backend.api.routes.health.STARTUP_TIME", "2026-03-31T18:10:00Z"), \ + patch("backend.api.routes.health._get_encoding_worker_manager", return_value=mock_manager): + + result = await system_status(auth=mock_auth) + + encoder = result["services"]["encoder"] + assert "admin_details" in encoder + admin = encoder["admin_details"] + assert admin["primary_vm"] == "encoding-worker-b" + assert admin["secondary_vm"] == "encoding-worker-a" + assert admin["last_swap_at"] == "2026-03-31T18:10:00Z" + assert admin["deploy_in_progress"] is False diff --git a/backend/version.py b/backend/version.py index d9addd87..5f936fed 100644 --- a/backend/version.py +++ b/backend/version.py @@ -25,3 +25,18 @@ def get_version() -> str: VERSION = get_version() + +# Build metadata — set by CI during deployment +COMMIT_SHA = os.environ.get("COMMIT_SHA", "") +PR_NUMBER = os.environ.get("PR_NUMBER", "") +PR_TITLE = os.environ.get("PR_TITLE", "") +STARTUP_TIME = None # Set on first import + + +def _get_startup_time() -> str: + """Record when the backend process started (proxy for deploy time).""" + from datetime import datetime, timezone + return datetime.now(timezone.utc).isoformat() + + +STARTUP_TIME = _get_startup_time() diff --git a/docs/superpowers/plans/2026-03-31-service-health-modal.md b/docs/superpowers/plans/2026-03-31-service-health-modal.md new file mode 100644 index 00000000..b09359b9 --- /dev/null +++ b/docs/superpowers/plans/2026-03-31-service-health-modal.md @@ -0,0 +1,1049 @@ +# Service Health Status Modal — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Replace the static footer version line with a clickable element that opens a modal showing all service health, versions, deploy info, and (for admins) blue-green encoder details with PR links. + +**Architecture:** New `/api/health/system-status` backend endpoint aggregates all service health + Firestore blue-green config into one response. Frontend gets a new `SystemStatusModal` component using the existing shadcn Dialog. Build metadata (git SHA, PR number/title) is baked in at CI time via env vars. + +**Tech Stack:** FastAPI, Firestore, Next.js, shadcn Dialog, Tailwind, GitHub Actions + +**Spec:** `docs/superpowers/specs/2026-03-31-service-health-modal-design.md` + +--- + +## File Map + +| Action | File | Responsibility | +|--------|------|---------------| +| Modify | `backend/version.py` | Add build metadata (commit SHA, PR number/title) | +| Modify | `backend/api/routes/health.py` | New `/api/health/system-status` endpoint | +| Create | `backend/tests/test_system_status_endpoint.py` | Unit tests for the new endpoint | +| Modify | `frontend/next.config.mjs` | Pass new env vars to frontend build | +| Modify | `frontend/lib/api.ts` | Add `SystemStatus` types and `getSystemStatus()` | +| Create | `frontend/components/system-status-modal.tsx` | The modal component | +| Modify | `frontend/components/version-footer.tsx` | Make clickable, open modal | +| Modify | `.github/workflows/ci.yml` | Capture git SHA + PR info, pass to builds | +| Modify | `backend/services/gce_encoding/main.py` | Add commit_sha to health response | + +--- + +### Task 1: Backend — Add build metadata to version module + +**Files:** +- Modify: `backend/version.py` +- Test: `backend/tests/test_system_status_endpoint.py` (created in Task 2) + +- [ ] **Step 1: Add build metadata env var readers to version.py** + +In `backend/version.py`, add after the existing `VERSION = get_version()` line: + +```python +# Build metadata — set by CI during deployment +COMMIT_SHA = os.environ.get("COMMIT_SHA", "") +PR_NUMBER = os.environ.get("PR_NUMBER", "") +PR_TITLE = os.environ.get("PR_TITLE", "") +STARTUP_TIME = None # Set on first import + + +def _get_startup_time() -> str: + """Record when the backend process started (proxy for deploy time).""" + from datetime import datetime, timezone + return datetime.now(timezone.utc).isoformat() + + +STARTUP_TIME = _get_startup_time() +``` + +- [ ] **Step 2: Commit** + +```bash +git add backend/version.py +git commit -m "feat: add build metadata env vars to version module" +``` + +--- + +### Task 2: Backend — New /api/health/system-status endpoint + +**Files:** +- Modify: `backend/api/routes/health.py` +- Create: `backend/tests/test_system_status_endpoint.py` + +- [ ] **Step 1: Write the failing test** + +Create `backend/tests/test_system_status_endpoint.py`: + +```python +"""Tests for the /api/health/system-status endpoint.""" +import pytest +from unittest.mock import patch, MagicMock, AsyncMock + + +@pytest.fixture +def mock_encoding_status(): + return { + "configured": True, + "enabled": True, + "available": True, + "status": "ok", + "active_jobs": 0, + "queue_length": 0, + "wheel_version": "0.155.3", + } + + +@pytest.fixture +def mock_flacfetch_status(): + return { + "configured": True, + "available": True, + "status": "ok", + "version": "0.19.1", + } + + +@pytest.fixture +def mock_separator_status(): + return { + "available": True, + "status": "ok", + "version": "0.4.3", + } + + +@pytest.fixture +def mock_encoding_worker_config(): + """Mock EncodingWorkerConfig dataclass.""" + config = MagicMock() + config.primary_vm = "encoding-worker-b" + config.primary_ip = "34.10.189.118" + config.primary_version = "0.155.3" + config.primary_deployed_at = "2026-03-31T18:10:00Z" + config.secondary_vm = "encoding-worker-a" + config.secondary_ip = "34.57.78.246" + config.secondary_version = "0.155.2" + config.secondary_deployed_at = "2026-03-29T14:00:00Z" + config.last_swap_at = "2026-03-31T18:10:00Z" + config.deploy_in_progress = False + config.deploy_in_progress_since = None + config.last_activity_at = None + return config + + +class TestSystemStatusPublic: + """Tests for unauthenticated /api/health/system-status.""" + + @pytest.mark.asyncio + async def test_returns_all_services( + self, mock_encoding_status, mock_flacfetch_status, mock_separator_status + ): + from backend.api.routes.health import system_status + + with patch("backend.api.routes.health.check_encoding_worker_status", new_callable=AsyncMock, return_value=mock_encoding_status), \ + patch("backend.api.routes.health.check_flacfetch_service_status", new_callable=AsyncMock, return_value=mock_flacfetch_status), \ + patch("backend.api.routes.health.check_audio_separator_status", return_value=mock_separator_status), \ + patch("backend.api.routes.health.VERSION", "0.155.3"), \ + patch("backend.api.routes.health.COMMIT_SHA", "8e2a949a"), \ + patch("backend.api.routes.health.PR_NUMBER", "644"), \ + patch("backend.api.routes.health.PR_TITLE", "fix: blue-green deploy"), \ + patch("backend.api.routes.health.STARTUP_TIME", "2026-03-31T18:10:00Z"): + + result = await system_status(auth=None) + + assert "services" in result + services = result["services"] + + # All 5 services present + assert "frontend" in services + assert "backend" in services + assert "encoder" in services + assert "flacfetch" in services + assert "separator" in services + + # Backend has build metadata + backend_svc = services["backend"] + assert backend_svc["version"] == "0.155.3" + assert backend_svc["commit_sha"] == "8e2a949a" + assert backend_svc["pr_number"] == "644" + + # Encoder has status + assert services["encoder"]["status"] == "ok" + assert services["encoder"]["version"] == "0.155.3" + + # No admin_details without auth + assert "admin_details" not in services["encoder"] + + @pytest.mark.asyncio + async def test_offline_service(self, mock_flacfetch_status, mock_separator_status): + from backend.api.routes.health import system_status + + offline_encoding = { + "configured": True, + "enabled": True, + "available": False, + "error": "Connection refused", + } + + with patch("backend.api.routes.health.check_encoding_worker_status", new_callable=AsyncMock, return_value=offline_encoding), \ + patch("backend.api.routes.health.check_flacfetch_service_status", new_callable=AsyncMock, return_value=mock_flacfetch_status), \ + patch("backend.api.routes.health.check_audio_separator_status", return_value=mock_separator_status), \ + patch("backend.api.routes.health.VERSION", "0.155.3"), \ + patch("backend.api.routes.health.COMMIT_SHA", ""), \ + patch("backend.api.routes.health.PR_NUMBER", ""), \ + patch("backend.api.routes.health.PR_TITLE", ""), \ + patch("backend.api.routes.health.STARTUP_TIME", "2026-03-31T18:10:00Z"): + + result = await system_status(auth=None) + + assert result["services"]["encoder"]["status"] == "offline" + + +class TestSystemStatusAdmin: + """Tests for admin-authenticated /api/health/system-status.""" + + @pytest.mark.asyncio + async def test_admin_gets_bluegreen_details( + self, mock_encoding_status, mock_flacfetch_status, mock_separator_status, mock_encoding_worker_config + ): + from backend.api.routes.health import system_status + from backend.services.auth_service import UserType + + mock_auth = ("token123", UserType.ADMIN, -1) + mock_manager = MagicMock() + mock_manager.get_config.return_value = mock_encoding_worker_config + + with patch("backend.api.routes.health.check_encoding_worker_status", new_callable=AsyncMock, return_value=mock_encoding_status), \ + patch("backend.api.routes.health.check_flacfetch_service_status", new_callable=AsyncMock, return_value=mock_flacfetch_status), \ + patch("backend.api.routes.health.check_audio_separator_status", return_value=mock_separator_status), \ + patch("backend.api.routes.health.VERSION", "0.155.3"), \ + patch("backend.api.routes.health.COMMIT_SHA", ""), \ + patch("backend.api.routes.health.PR_NUMBER", ""), \ + patch("backend.api.routes.health.PR_TITLE", ""), \ + patch("backend.api.routes.health.STARTUP_TIME", "2026-03-31T18:10:00Z"), \ + patch("backend.api.routes.health._get_encoding_worker_manager", return_value=mock_manager): + + result = await system_status(auth=mock_auth) + + encoder = result["services"]["encoder"] + assert "admin_details" in encoder + admin = encoder["admin_details"] + assert admin["primary_vm"] == "encoding-worker-b" + assert admin["secondary_vm"] == "encoding-worker-a" + assert admin["last_swap_at"] == "2026-03-31T18:10:00Z" + assert admin["deploy_in_progress"] is False +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-fix-bluegreen-deploy && python -m pytest backend/tests/test_system_status_endpoint.py -v` +Expected: FAIL — `system_status` function does not exist yet. + +- [ ] **Step 3: Implement the endpoint** + +Add these imports to the top of `backend/api/routes/health.py` (after existing imports): + +```python +from backend.version import VERSION, COMMIT_SHA, PR_NUMBER, PR_TITLE, STARTUP_TIME +from backend.api.dependencies import optional_auth +from backend.services.auth_service import UserType +``` + +Remove the existing `from backend.version import VERSION` line (now covered by the expanded import). + +Add a helper to get the encoding worker manager (for testability): + +```python +def _get_encoding_worker_manager(): + """Get the EncodingWorkerManager instance. Separated for testability.""" + try: + encoding_service = get_encoding_service() + if hasattr(encoding_service, '_worker_manager') and encoding_service._worker_manager: + return encoding_service._worker_manager + except Exception: + pass + return None +``` + +Add the endpoint (before the existing `@router.get("/health/detailed")` function): + +```python +@router.get("/health/system-status") +async def system_status( + auth: Optional[Tuple] = Depends(optional_auth), +) -> Dict[str, Any]: + """ + Aggregated system status for all services. + + Returns basic info for everyone. Admin users get additional + blue-green deployment details for the encoder. + """ + is_admin = auth is not None and len(auth) >= 2 and auth[1] == UserType.ADMIN + + # Fetch all health statuses concurrently + import asyncio + encoding_task = check_encoding_worker_status() + flacfetch_task = check_flacfetch_service_status() + encoding_status, flacfetch_status = await asyncio.gather( + encoding_task, flacfetch_task + ) + separator_status = check_audio_separator_status() + + # Build frontend service info (from env vars baked at build time — returned as-is) + frontend_svc = { + "status": "ok", + "version": VERSION, # Frontend and backend share the same version from pyproject.toml + } + + # Build backend service info + backend_svc = { + "status": "ok", + "version": VERSION, + "deployed_at": STARTUP_TIME, + } + if COMMIT_SHA: + backend_svc["commit_sha"] = COMMIT_SHA + if PR_NUMBER: + backend_svc["pr_number"] = PR_NUMBER + if PR_TITLE: + backend_svc["pr_title"] = PR_TITLE + + # Build encoder service info + encoder_svc = { + "status": "ok" if encoding_status.get("available") else "offline", + "version": encoding_status.get("wheel_version"), + "active_jobs": encoding_status.get("active_jobs", 0), + } + + # Build flacfetch service info + flacfetch_svc = { + "status": "ok" if flacfetch_status.get("available") else "offline", + "version": flacfetch_status.get("version"), + } + + # Build separator service info + separator_svc = { + "status": "ok" if separator_status.get("available") else "offline", + "version": separator_status.get("version"), + } + + # Admin-only: blue-green encoder details from Firestore + if is_admin: + manager = _get_encoding_worker_manager() + if manager: + try: + config = manager.get_config() + encoder_svc["admin_details"] = { + "primary_vm": config.primary_vm, + "primary_ip": config.primary_ip, + "primary_version": config.primary_version, + "primary_deployed_at": config.primary_deployed_at, + "secondary_vm": config.secondary_vm, + "secondary_ip": config.secondary_ip, + "secondary_version": config.secondary_version, + "secondary_deployed_at": config.secondary_deployed_at, + "last_swap_at": config.last_swap_at, + "deploy_in_progress": config.deploy_in_progress, + "active_jobs": encoding_status.get("active_jobs", 0), + "queue_length": encoding_status.get("queue_length", 0), + } + except Exception as e: + logger.warning(f"Failed to get blue-green config: {e}") + + # Add error details for offline services + if not flacfetch_status.get("available"): + flacfetch_svc["admin_details"] = {"error": flacfetch_status.get("error")} + if not separator_status.get("available"): + separator_svc["admin_details"] = {"error": separator_status.get("error")} + if not encoding_status.get("available") and "admin_details" not in encoder_svc: + encoder_svc["admin_details"] = {"error": encoding_status.get("error")} + + return { + "services": { + "frontend": frontend_svc, + "backend": backend_svc, + "encoder": encoder_svc, + "flacfetch": flacfetch_svc, + "separator": separator_svc, + } + } +``` + +Also add `Tuple` to the `typing` import at the top of the file (add to existing `from typing import Dict, Any, Optional` line): + +```python +from typing import Dict, Any, Optional, Tuple +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-fix-bluegreen-deploy && python -m pytest backend/tests/test_system_status_endpoint.py -v` +Expected: All 3 tests PASS. + +- [ ] **Step 5: Commit** + +```bash +git add backend/api/routes/health.py backend/tests/test_system_status_endpoint.py +git commit -m "feat: add /api/health/system-status endpoint with admin blue-green details" +``` + +--- + +### Task 3: Frontend — Add API types and fetch function + +**Files:** +- Modify: `frontend/lib/api.ts` + +- [ ] **Step 1: Add SystemStatus types** + +Add after the existing `AudioSeparatorHealth` interface (around line 305): + +```typescript +export interface ServiceStatus { + status: string; // 'ok' | 'offline' + version?: string | null; + deployed_at?: string | null; + commit_sha?: string | null; + pr_number?: string | null; + pr_title?: string | null; + active_jobs?: number; + admin_details?: { + // Encoder blue-green + primary_vm?: string; + primary_ip?: string; + primary_version?: string; + primary_deployed_at?: string; + secondary_vm?: string; + secondary_ip?: string; + secondary_version?: string; + secondary_deployed_at?: string; + last_swap_at?: string; + deploy_in_progress?: boolean; + active_jobs?: number; + queue_length?: number; + // Error details for offline services + error?: string | null; + }; +} + +export interface SystemStatus { + services: { + frontend: ServiceStatus; + backend: ServiceStatus; + encoder: ServiceStatus; + flacfetch: ServiceStatus; + separator: ServiceStatus; + }; +} +``` + +- [ ] **Step 2: Add getSystemStatus method** + +Add to the `api` object (after the existing `getAudioSeparatorHealth` method): + +```typescript + /** + * Get aggregated system status for all services + */ + async getSystemStatus(): Promise { + const response = await fetch(`${API_BASE_URL}/api/health/system-status`, { + headers: getAuthHeaders(), + }); + return handleResponse(response); + }, +``` + +- [ ] **Step 3: Commit** + +```bash +git add frontend/lib/api.ts +git commit -m "feat: add SystemStatus types and getSystemStatus API method" +``` + +--- + +### Task 4: Frontend — Create SystemStatusModal component + +**Files:** +- Create: `frontend/components/system-status-modal.tsx` + +- [ ] **Step 1: Create the modal component** + +Create `frontend/components/system-status-modal.tsx`: + +```tsx +"use client"; + +import { useEffect, useState } from "react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { api, SystemStatus, ServiceStatus } from "@/lib/api"; + +interface SystemStatusModalProps { + open: boolean; + onClose: () => void; +} + +function formatRelativeTime(isoString: string | null | undefined): string { + if (!isoString) return ""; + const date = new Date(isoString); + const now = new Date(); + const diffMs = now.getTime() - date.getTime(); + const diffMin = Math.floor(diffMs / 60000); + if (diffMin < 1) return "just now"; + if (diffMin < 60) return `${diffMin}m ago`; + const diffHours = Math.floor(diffMin / 60); + if (diffHours < 24) return `${diffHours}h ago`; + const diffDays = Math.floor(diffHours / 24); + return `${diffDays}d ago`; +} + +function truncate(str: string, maxLen: number): string { + if (str.length <= maxLen) return str; + return str.slice(0, maxLen - 1) + "…"; +} + +function StatusDot({ status }: { status: string }) { + const isOk = status === "ok"; + return ( + + {isOk ? "●" : "○"} + + ); +} + +function ServiceCard({ + name, + service, + children, +}: { + name: string; + service: ServiceStatus; + children?: React.ReactNode; +}) { + const isOk = service.status === "ok"; + return ( +
+
+
+ + {name} +
+ + {isOk ? "Healthy" : "Offline"} + +
+ + {service.version && ( +
v{service.version}
+ )} + + {service.deployed_at && ( +
+ Deployed {formatRelativeTime(service.deployed_at)} +
+ )} + + {service.pr_number && ( +
+ + #{service.pr_number} + + {service.pr_title && ( + + {" — "} + {truncate(service.pr_title, 50)} + + )} +
+ )} + + {children} +
+ ); +} + +function BlueGreenSection({ details }: { details: NonNullable }) { + const [expanded, setExpanded] = useState(false); + + return ( +
+ + + {expanded && ( +
+ {/* Primary */} +
+
+ Primary +
+
{details.primary_vm}
+
v{details.primary_version}
+ {details.primary_deployed_at && ( +
+ {formatRelativeTime(details.primary_deployed_at)} +
+ )} +
+ + {/* Secondary */} +
+
+ Secondary (stopped) +
+
{details.secondary_vm}
+ {details.secondary_version && ( +
v{details.secondary_version}
+ )} +
+ + {/* Metadata row */} +
+ {details.last_swap_at && ( + Last swap: {formatRelativeTime(details.last_swap_at)} + )} + {details.active_jobs != null && details.active_jobs > 0 && ( + Active jobs: {details.active_jobs} + )} +
+
+ )} +
+ ); +} + +export function SystemStatusModal({ open, onClose }: SystemStatusModalProps) { + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + if (!open) return; + + setLoading(true); + setError(null); + + api + .getSystemStatus() + .then(setStatus) + .catch((e) => setError(e.message || "Failed to fetch status")) + .finally(() => setLoading(false)); + }, [open]); + + const frontendVersion = process.env.NEXT_PUBLIC_APP_VERSION || "dev"; + const frontendCommitSha = process.env.NEXT_PUBLIC_COMMIT_SHA || ""; + const frontendPrNumber = process.env.NEXT_PUBLIC_PR_NUMBER || ""; + const frontendPrTitle = process.env.NEXT_PUBLIC_PR_TITLE || ""; + const frontendBuildTime = process.env.NEXT_PUBLIC_BUILD_TIME || ""; + + // Override frontend service with build-time values + const services = status?.services; + const frontendService: ServiceStatus = { + status: "ok", + version: frontendVersion, + deployed_at: frontendBuildTime || undefined, + commit_sha: frontendCommitSha || undefined, + pr_number: frontendPrNumber || undefined, + pr_title: frontendPrTitle || undefined, + }; + + return ( + !isOpen && onClose()}> + + + System Status + + + {loading && ( +
Loading...
+ )} + + {error && ( +
{error}
+ )} + + {services && !loading && ( +
+ + + + {services.encoder.admin_details && + services.encoder.admin_details.primary_vm && ( + + )} + + + +
+ )} +
+
+ ); +} +``` + +- [ ] **Step 2: Commit** + +```bash +git add frontend/components/system-status-modal.tsx +git commit -m "feat: add SystemStatusModal component with blue-green details" +``` + +--- + +### Task 5: Frontend — Make footer clickable and wire up modal + +**Files:** +- Modify: `frontend/components/version-footer.tsx` + +- [ ] **Step 1: Update version-footer to open modal on click** + +Replace the entire content of `frontend/components/version-footer.tsx` with: + +```tsx +"use client"; + +import { useEffect, useState } from "react"; +import { api, EncodingWorkerHealth, FlacfetchHealth, AudioSeparatorHealth } from "@/lib/api"; +import { SystemStatusModal } from "./system-status-modal"; + +export function VersionFooter() { + const [backendVersion, setBackendVersion] = useState(null); + const [encodingWorker, setEncodingWorker] = useState(null); + const [flacfetch, setFlacfetch] = useState(null); + const [audioSeparator, setAudioSeparator] = useState(null); + const [modalOpen, setModalOpen] = useState(false); + // Frontend version is injected at build time from pyproject.toml + const frontendVersion = process.env.NEXT_PUBLIC_APP_VERSION || "dev"; + + useEffect(() => { + const fetchBackendVersion = async () => { + try { + const info = await api.getBackendInfo(); + setBackendVersion(info.version); + } catch (error) { + console.error("Failed to fetch backend version:", error); + setBackendVersion(null); + } + }; + + const fetchEncodingWorkerHealth = async () => { + try { + const health = await api.getEncodingWorkerHealth(); + setEncodingWorker(health); + } catch (error) { + console.error("Failed to fetch encoding worker health:", error); + setEncodingWorker(null); + } + }; + + const fetchFlacfetchHealth = async () => { + try { + const health = await api.getFlacfetchHealth(); + setFlacfetch(health); + } catch (error) { + console.error("Failed to fetch flacfetch health:", error); + setFlacfetch(null); + } + }; + + const fetchAudioSeparatorHealth = async () => { + try { + const health = await api.getAudioSeparatorHealth(); + setAudioSeparator(health); + } catch (error) { + console.error("Failed to fetch audio separator health:", error); + setAudioSeparator(null); + } + }; + + fetchBackendVersion(); + fetchEncodingWorkerHealth(); + fetchFlacfetchHealth(); + fetchAudioSeparatorHealth(); + }, []); + + // Render encoding worker status indicator + const renderEncodingWorkerStatus = () => { + if (!encodingWorker) return null; + if (encodingWorker.status === "not_configured") return null; + + const isHealthy = encodingWorker.available && encodingWorker.status === "ok"; + const statusColor = isHealthy ? "text-green-500" : "text-red-500"; + const statusDot = isHealthy ? "●" : "○"; + + let statusText = isHealthy ? "Encoder" : "Encoder offline"; + if (isHealthy && encodingWorker.version) { + statusText = `Encoder: v${encodingWorker.version}`; + } + if (isHealthy && encodingWorker.active_jobs && encodingWorker.active_jobs > 0) { + statusText += ` (${encodingWorker.active_jobs} active)`; + } + + return ( + <> + {" | "} + {statusDot} {statusText} + + ); + }; + + // Render flacfetch status indicator + const renderFlacfetchStatus = () => { + if (!flacfetch) return null; + if (flacfetch.status === "not_configured") return null; + + const isHealthy = flacfetch.available && flacfetch.status === "ok"; + const statusColor = isHealthy ? "text-green-500" : "text-red-500"; + const statusDot = isHealthy ? "●" : "○"; + + let statusText = isHealthy ? "Flacfetch" : "Flacfetch offline"; + if (isHealthy && flacfetch.version) { + statusText = `Flacfetch: v${flacfetch.version}`; + } + + return ( + <> + {" | "} + {statusDot} {statusText} + + ); + }; + + // Render audio separator status indicator + const renderAudioSeparatorStatus = () => { + if (!audioSeparator) return null; + if (audioSeparator.status === "not_configured") return null; + + const isHealthy = audioSeparator.available && audioSeparator.status === "ok"; + const statusColor = isHealthy ? "text-green-500" : "text-red-500"; + const statusDot = isHealthy ? "●" : "○"; + + let statusText = isHealthy ? "Separator" : "Separator offline"; + if (isHealthy && audioSeparator.version) { + statusText = `Separator: v${audioSeparator.version}`; + } + + return ( + <> + {" | "} + {statusDot} {statusText} + + ); + }; + + return ( + <> +
+
+

+ Powered by{" "} + + karaoke-gen + +

+

setModalOpen(true)} + title="Click for detailed system status" + > + Frontend: v{frontendVersion} + {backendVersion && ( + <> + {" | "} + Backend: v{backendVersion} + + )} + {renderEncodingWorkerStatus()} + {renderFlacfetchStatus()} + {renderAudioSeparatorStatus()} +

+
+
+ + setModalOpen(false)} + /> + + ); +} +``` + +- [ ] **Step 2: Commit** + +```bash +git add frontend/components/version-footer.tsx +git commit -m "feat: make footer version line clickable to open system status modal" +``` + +--- + +### Task 6: Frontend — Pass build metadata env vars in next.config.mjs + +**Files:** +- Modify: `frontend/next.config.mjs` + +- [ ] **Step 1: Add new env vars to next.config.mjs** + +In `frontend/next.config.mjs`, add to the `env` object (after the `NEXT_PUBLIC_APP_VERSION` line): + +```javascript + env: { + NEXT_PUBLIC_APP_VERSION: appVersion, + NEXT_PUBLIC_COMMIT_SHA: process.env.NEXT_PUBLIC_COMMIT_SHA || '', + NEXT_PUBLIC_PR_NUMBER: process.env.NEXT_PUBLIC_PR_NUMBER || '', + NEXT_PUBLIC_PR_TITLE: process.env.NEXT_PUBLIC_PR_TITLE || '', + NEXT_PUBLIC_BUILD_TIME: process.env.NEXT_PUBLIC_BUILD_TIME || new Date().toISOString(), + }, +``` + +- [ ] **Step 2: Commit** + +```bash +git add frontend/next.config.mjs +git commit -m "feat: pass build metadata env vars (commit, PR, build time) to frontend" +``` + +--- + +### Task 7: CI — Capture git metadata and pass to builds + +**Files:** +- Modify: `.github/workflows/ci.yml` + +- [ ] **Step 1: Find the deploy job and add build metadata step** + +In `.github/workflows/ci.yml`, in the deploy job (the one that runs on merge to main), add a step early in the job (after checkout and before any build/deploy steps) to capture git info: + +```yaml + - name: Capture build metadata + id: build-meta + run: | + echo "sha=${GITHUB_SHA::8}" >> $GITHUB_OUTPUT + # For squash-merge commits, PR number is in the commit message as (#NNN) + PR_NUM=$(git log -1 --format=%s | grep -oP '\(#\K[0-9]+' | head -1) + if [ -n "$PR_NUM" ]; then + PR_TITLE=$(gh pr view "$PR_NUM" --json title -q .title 2>/dev/null || echo "") + else + PR_NUM="" + PR_TITLE="" + fi + echo "pr_number=${PR_NUM}" >> $GITHUB_OUTPUT + echo "pr_title=${PR_TITLE}" >> $GITHUB_OUTPUT + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} +``` + +- [ ] **Step 2: Pass metadata to frontend build** + +Find the frontend build/deploy step (Cloudflare Pages deploy) and add env vars: + +```yaml + env: + NEXT_PUBLIC_COMMIT_SHA: ${{ steps.build-meta.outputs.sha }} + NEXT_PUBLIC_PR_NUMBER: ${{ steps.build-meta.outputs.pr_number }} + NEXT_PUBLIC_PR_TITLE: ${{ steps.build-meta.outputs.pr_title }} + NEXT_PUBLIC_BUILD_TIME: ${{ github.event.head_commit.timestamp }} +``` + +- [ ] **Step 3: Pass metadata to Cloud Run deploy** + +Find the Cloud Run deploy step and add env vars to the service: + +```yaml + # Add to the gcloud run deploy command's --set-env-vars flag: + COMMIT_SHA=${{ steps.build-meta.outputs.sha }},\ + PR_NUMBER=${{ steps.build-meta.outputs.pr_number }},\ + PR_TITLE=${{ steps.build-meta.outputs.pr_title }} +``` + +Note: Find the exact `gcloud run deploy` command in the CI file and add these to its existing `--set-env-vars` flag. If it uses `--env-vars-file`, add them there instead. The exact integration depends on how the existing deploy step is structured — read the file and match the pattern. + +- [ ] **Step 4: Commit** + +```bash +git add .github/workflows/ci.yml +git commit -m "ci: capture git SHA and PR info, pass to frontend and backend deploys" +``` + +--- + +### Task 8: Run all tests and verify + +- [ ] **Step 1: Run backend tests** + +```bash +cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-fix-bluegreen-deploy +python -m pytest backend/tests/test_system_status_endpoint.py -v +``` + +Expected: All tests pass. + +- [ ] **Step 2: Run frontend build to verify no TypeScript errors** + +```bash +cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-fix-bluegreen-deploy/frontend +npx next build 2>&1 | tail -20 +``` + +Expected: Build succeeds without errors. + +- [ ] **Step 3: Run full test suite** + +```bash +cd /Users/andrew/Projects/nomadkaraoke/karaoke-gen-fix-bluegreen-deploy +make test 2>&1 | tail -500 +``` + +Expected: All tests pass. + +- [ ] **Step 4: Bump version** + +```bash +# Bump patch version in pyproject.toml +# Current version visible in pyproject.toml — increment the patch number +``` + +- [ ] **Step 5: Final commit** + +```bash +git add pyproject.toml +git commit -m "chore: bump version to " +``` diff --git a/docs/superpowers/specs/2026-03-31-service-health-modal-design.md b/docs/superpowers/specs/2026-03-31-service-health-modal-design.md new file mode 100644 index 00000000..5fd849d0 --- /dev/null +++ b/docs/superpowers/specs/2026-03-31-service-health-modal-design.md @@ -0,0 +1,213 @@ +# Service Health Status Modal — Design Spec + +## Overview + +Replace the static footer version/status line with a clickable element that opens a modal showing detailed system status. Non-admin users see service health, versions, and deploy info. Admin users additionally see blue-green encoder deployment details. Each service links to the PR that produced its current version. + +## Footer Changes + +The existing footer line (`Frontend: v0.155.3 | Backend: v0.155.3 | ● Encoder: v0.155.3 | ...`) becomes a single clickable element. On hover: `cursor: pointer`, slight opacity change. No other visual changes to the footer layout. + +Clicking anywhere on the version/status line opens the modal. + +## Modal — Public View (everyone) + +### Layout + +- Standard modal: dark backdrop (click to dismiss), X close button, title "System Status" +- Responsive card grid: 2 columns on desktop, 1 column on mobile +- 5 service cards: **Frontend**, **Backend**, **Encoder**, **Flacfetch**, **Separator** + +### Card Contents + +Each card shows: + +| Element | Example | +|---------|---------| +| Status indicator | Green dot (●) + "Healthy" or red dot (○) + "Offline" | +| Version | `v0.155.3` | +| Deploy time | "Deployed 2 hours ago" (relative) | +| Latest change | `#644 — fix blue-green deploy` (clickable link to GitHub PR) | + +The "Latest change" line links to the GitHub PR that produced the currently deployed version. If unavailable, this line is omitted. + +## Modal — Admin View (additional detail) + +When the user has an admin token (stored in localStorage from login), the modal request includes it and the response includes extra data. + +### Encoder Blue-Green Section + +Inside the Encoder card, below the standard info, an expandable section labeled "Blue-Green Deployment" shows: + +- **Two side-by-side mini-cards**: Primary (green-tinted border) and Secondary (muted/grey) +- Each mini-card shows: + - VM name (e.g., `encoding-worker-b`) + - Version (e.g., `v0.155.3`) + - Role label: "PRIMARY" (green) or "SECONDARY (stopped)" (muted) +- **Deploy status line**: "Last swap: 2 hours ago" or "⚠ Deploy in progress..." (amber) +- **Active jobs**: count on primary, if > 0 + +### Admin Detail on Other Cards + +For Backend, Flacfetch, and Separator cards, admin view additionally shows: +- Error message if status is offline (from the health check error field) + +No extra admin detail needed for Frontend (it's a static site). + +## New Backend Endpoint + +### `GET /api/health/system-status` + +Single endpoint that aggregates all service health data. Avoids the current pattern of 4 separate fetches. + +**Authentication**: Optional `X-Admin-Token` header. If valid admin token, response includes `admin_details`. If omitted or invalid, those fields are absent. + +**Response (public fields)**: + +```json +{ + "services": { + "frontend": { + "status": "ok", + "version": "0.155.3", + "deployed_at": "2026-03-31T18:10:00Z", + "commit_sha": "8e2a949a", + "pr_number": 644, + "pr_title": "fix: complete blue-green deploy test payload and status check" + }, + "backend": { + "status": "ok", + "version": "0.155.3", + "deployed_at": "2026-03-31T18:10:00Z", + "commit_sha": "8e2a949a", + "pr_number": 644, + "pr_title": "fix: complete blue-green deploy test payload and status check" + }, + "encoder": { + "status": "ok", + "version": "0.155.3", + "deployed_at": "2026-03-31T18:10:00Z", + "active_jobs": 0 + }, + "flacfetch": { + "status": "ok", + "version": "0.19.1" + }, + "separator": { + "status": "ok", + "version": "0.4.3" + } + } +} +``` + +**Additional fields when admin token is valid**: + +```json +{ + "services": { + "encoder": { + "admin_details": { + "primary_vm": "encoding-worker-b", + "primary_ip": "34.10.189.118", + "primary_version": "0.155.3", + "primary_deployed_at": "2026-03-31T18:10:00Z", + "secondary_vm": "encoding-worker-a", + "secondary_ip": "34.57.78.246", + "secondary_version": "0.155.2", + "secondary_deployed_at": "2026-03-29T14:00:00Z", + "last_swap_at": "2026-03-31T18:10:00Z", + "deploy_in_progress": false, + "active_jobs": 0, + "queue_length": 0 + } + }, + "flacfetch": { + "admin_details": { + "error": null + } + }, + "separator": { + "admin_details": { + "error": null + } + } + } +} +``` + +## Build Metadata — Git SHA and PR Info + +Currently no git metadata is baked into builds. We need to add this. + +### CI Changes (`.github/workflows/ci.yml`) + +Add a step early in the deploy job that captures git info: + +```yaml +- name: Capture build metadata + id: build-meta + run: | + echo "sha=${GITHUB_SHA::8}" >> $GITHUB_OUTPUT + # For merge commits, extract PR number from commit message + PR_NUM=$(git log -1 --format=%s | grep -oP '#\K[0-9]+' | head -1) + PR_TITLE=$(gh pr view "$PR_NUM" --json title -q .title 2>/dev/null || echo "") + echo "pr_number=${PR_NUM}" >> $GITHUB_OUTPUT + echo "pr_title=${PR_TITLE}" >> $GITHUB_OUTPUT +``` + +### Where metadata is stored + +| Service | Storage mechanism | +|---------|------------------| +| **Frontend** | New env vars `NEXT_PUBLIC_COMMIT_SHA`, `NEXT_PUBLIC_PR_NUMBER`, `NEXT_PUBLIC_PR_TITLE`, `NEXT_PUBLIC_BUILD_TIME` set in CI before `npm run build`. Baked into the static bundle at build time via `next.config.mjs`. The `deployed_at` for frontend is the build timestamp (close enough to deploy time for Cloudflare Pages). | +| **Backend** | New env vars `COMMIT_SHA`, `PR_NUMBER`, `PR_TITLE` set on the Cloud Run service. Read at runtime in `backend/version.py`. | +| **Encoder** | Written to GCS version manifest alongside the existing `version.txt`. The GCE worker reads it on startup and includes in `/health` response. | +| **Flacfetch / Separator** | These are separate repos with their own CI. PR info can be added later — for now, omit the "Latest change" line for these services. | + +### GitHub PR Link Format + +The frontend constructs the PR link as: +``` +https://github.com/nomadkaraoke/karaoke-gen/pull/{pr_number} +``` + +Displayed as: `#{pr_number} — {pr_title}` (truncated to ~60 chars if needed). + +## Frontend Components + +### New files +- `frontend/components/system-status-modal.tsx` — the modal component + +### Modified files +- `frontend/components/version-footer.tsx` — make status line clickable, add modal toggle state +- `frontend/lib/api.ts` — add `SystemStatus` types and `getSystemStatus()` method +- `frontend/next.config.mjs` — pass through new `NEXT_PUBLIC_COMMIT_SHA`, `NEXT_PUBLIC_PR_NUMBER`, `NEXT_PUBLIC_PR_TITLE` env vars + +### Modified backend files +- `backend/api/routes/health.py` — new `/api/health/system-status` endpoint +- `backend/version.py` — read `COMMIT_SHA`, `PR_NUMBER`, `PR_TITLE` env vars +- `.github/workflows/ci.yml` — capture build metadata, pass to builds/deploys + +### Data Flow + +1. User clicks footer → modal opens, state set to loading +2. Modal calls `GET /api/health/system-status` (with admin token header if available in localStorage) +3. Endpoint aggregates: existing health checks + Firestore blue-green config + build metadata env vars +4. Frontend renders cards from response +5. No auto-refresh — data is fresh on each open + +## Styling + +- Follow existing project design patterns (Tailwind, CSS variables for theming) +- Modal uses the same dark theme as the rest of the app +- Cards use `var(--card-border)`, `var(--card-bg)` etc. for consistency +- Green: `text-green-500` for healthy, red: `text-red-500` for offline +- Blue-green section uses amber accent (`text-amber-500`, `border-amber-500/30`) to distinguish it as an operational detail + +## Out of Scope + +- Auto-refresh / polling in the modal (can be added later) +- Flacfetch and Separator PR info (separate repos, separate CI) +- Historical deploy data or deploy log +- Notifications or alerts for status changes diff --git a/frontend/components/system-status-modal.tsx b/frontend/components/system-status-modal.tsx new file mode 100644 index 00000000..02b46f46 --- /dev/null +++ b/frontend/components/system-status-modal.tsx @@ -0,0 +1,244 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { api, SystemStatus, ServiceStatus } from "@/lib/api"; + +interface SystemStatusModalProps { + open: boolean; + onClose: () => void; +} + +function formatRelativeTime(isoString: string | null | undefined): string { + if (!isoString) return ""; + const date = new Date(isoString); + const now = new Date(); + const diffMs = now.getTime() - date.getTime(); + const diffMin = Math.floor(diffMs / 60000); + if (diffMin < 1) return "just now"; + if (diffMin < 60) return `${diffMin}m ago`; + const diffHours = Math.floor(diffMin / 60); + if (diffHours < 24) return `${diffHours}h ago`; + const diffDays = Math.floor(diffHours / 24); + return `${diffDays}d ago`; +} + +function truncate(str: string, maxLen: number): string { + if (str.length <= maxLen) return str; + return str.slice(0, maxLen - 1) + "\u2026"; +} + +function StatusDot({ status }: { status: string }) { + const isOk = status === "ok"; + return ( + + {isOk ? "\u25CF" : "\u25CB"} + + ); +} + +function ServiceCard({ + name, + service, + children, +}: { + name: string; + service: ServiceStatus; + children?: React.ReactNode; +}) { + const isOk = service.status === "ok"; + return ( +
+
+
+ + {name} +
+ + {isOk ? "Healthy" : "Offline"} + +
+ + {service.version && ( +
v{service.version}
+ )} + + {service.deployed_at && ( +
+ Deployed {formatRelativeTime(service.deployed_at)} +
+ )} + + {service.pr_number && ( +
+ + #{service.pr_number} + + {service.pr_title && ( + + {" \u2014 "} + {truncate(service.pr_title, 50)} + + )} +
+ )} + + {children} +
+ ); +} + +function BlueGreenSection({ details }: { details: NonNullable }) { + const [expanded, setExpanded] = useState(false); + + return ( +
+ + + {expanded && ( +
+ {/* Primary */} +
+
+ Primary +
+
{details.primary_vm}
+
v{details.primary_version}
+ {details.primary_deployed_at && ( +
+ {formatRelativeTime(details.primary_deployed_at)} +
+ )} +
+ + {/* Secondary */} +
+
+ Secondary (stopped) +
+
{details.secondary_vm}
+ {details.secondary_version && ( +
v{details.secondary_version}
+ )} +
+ + {/* Metadata row */} +
+ {details.last_swap_at && ( + Last swap: {formatRelativeTime(details.last_swap_at)} + )} + {details.active_jobs != null && details.active_jobs > 0 && ( + Active jobs: {details.active_jobs} + )} +
+
+ )} +
+ ); +} + +export function SystemStatusModal({ open, onClose }: SystemStatusModalProps) { + const [status, setStatus] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + if (!open) return; + + setLoading(true); + setError(null); + + api + .getSystemStatus() + .then(setStatus) + .catch((e) => setError(e.message || "Failed to fetch status")) + .finally(() => setLoading(false)); + }, [open]); + + const frontendVersion = process.env.NEXT_PUBLIC_APP_VERSION || "dev"; + const frontendCommitSha = process.env.NEXT_PUBLIC_COMMIT_SHA || ""; + const frontendPrNumber = process.env.NEXT_PUBLIC_PR_NUMBER || ""; + const frontendPrTitle = process.env.NEXT_PUBLIC_PR_TITLE || ""; + const frontendBuildTime = process.env.NEXT_PUBLIC_BUILD_TIME || ""; + + // Override frontend service with build-time values + const services = status?.services; + const frontendService: ServiceStatus = { + status: "ok", + version: frontendVersion, + deployed_at: frontendBuildTime || undefined, + commit_sha: frontendCommitSha || undefined, + pr_number: frontendPrNumber || undefined, + pr_title: frontendPrTitle || undefined, + }; + + return ( + !isOpen && onClose()}> + + + System Status + + + {loading && ( +
Loading...
+ )} + + {error && ( +
{error}
+ )} + + {services && !loading && ( +
+ + + + {services.encoder.admin_details && + services.encoder.admin_details.primary_vm && ( + + )} + + + +
+ )} +
+
+ ); +} diff --git a/frontend/components/version-footer.tsx b/frontend/components/version-footer.tsx index b0e84214..fec5d95e 100644 --- a/frontend/components/version-footer.tsx +++ b/frontend/components/version-footer.tsx @@ -2,13 +2,14 @@ import { useEffect, useState } from "react"; import { api, EncodingWorkerHealth, FlacfetchHealth, AudioSeparatorHealth } from "@/lib/api"; +import { SystemStatusModal } from "./system-status-modal"; export function VersionFooter() { const [backendVersion, setBackendVersion] = useState(null); const [encodingWorker, setEncodingWorker] = useState(null); const [flacfetch, setFlacfetch] = useState(null); const [audioSeparator, setAudioSeparator] = useState(null); - // Frontend version is injected at build time from pyproject.toml + const [modalOpen, setModalOpen] = useState(false); const frontendVersion = process.env.NEXT_PUBLIC_APP_VERSION || "dev"; useEffect(() => { @@ -58,27 +59,18 @@ export function VersionFooter() { fetchAudioSeparatorHealth(); }, []); - // Render encoding worker status indicator const renderEncodingWorkerStatus = () => { - if (!encodingWorker) { - return null; - } - - if (encodingWorker.status === "not_configured") { - return null; // Don't show if not configured - } + if (!encodingWorker) return null; + if (encodingWorker.status === "not_configured") return null; const isHealthy = encodingWorker.available && encodingWorker.status === "ok"; const statusColor = isHealthy ? "text-green-500" : "text-red-500"; - const statusDot = isHealthy ? "●" : "○"; + const statusDot = isHealthy ? "\u25CF" : "\u25CB"; - // Build status text let statusText = isHealthy ? "Encoder" : "Encoder offline"; if (isHealthy && encodingWorker.version) { statusText = `Encoder: v${encodingWorker.version}`; } - - // Add active jobs indicator if there are any if (isHealthy && encodingWorker.active_jobs && encodingWorker.active_jobs > 0) { statusText += ` (${encodingWorker.active_jobs} active)`; } @@ -86,29 +78,19 @@ export function VersionFooter() { return ( <> {" | "} - - {statusDot} - {" "} - {statusText} + {statusDot} {statusText} ); }; - // Render flacfetch status indicator const renderFlacfetchStatus = () => { - if (!flacfetch) { - return null; - } - - if (flacfetch.status === "not_configured") { - return null; // Don't show if not configured - } + if (!flacfetch) return null; + if (flacfetch.status === "not_configured") return null; const isHealthy = flacfetch.available && flacfetch.status === "ok"; const statusColor = isHealthy ? "text-green-500" : "text-red-500"; - const statusDot = isHealthy ? "●" : "○"; + const statusDot = isHealthy ? "\u25CF" : "\u25CB"; - // Build status text let statusText = isHealthy ? "Flacfetch" : "Flacfetch offline"; if (isHealthy && flacfetch.version) { statusText = `Flacfetch: v${flacfetch.version}`; @@ -117,27 +99,18 @@ export function VersionFooter() { return ( <> {" | "} - - {statusDot} - {" "} - {statusText} + {statusDot} {statusText} ); }; - // Render audio separator status indicator const renderAudioSeparatorStatus = () => { - if (!audioSeparator) { - return null; - } - - if (audioSeparator.status === "not_configured") { - return null; - } + if (!audioSeparator) return null; + if (audioSeparator.status === "not_configured") return null; const isHealthy = audioSeparator.available && audioSeparator.status === "ok"; const statusColor = isHealthy ? "text-green-500" : "text-red-500"; - const statusDot = isHealthy ? "●" : "○"; + const statusDot = isHealthy ? "\u25CF" : "\u25CB"; let statusText = isHealthy ? "Separator" : "Separator offline"; if (isHealthy && audioSeparator.version) { @@ -147,36 +120,44 @@ export function VersionFooter() { return ( <> {" | "} - - {statusDot} - {" "} - {statusText} + {statusDot} {statusText} ); }; return ( -
-
-

- Powered by{" "} - - karaoke-gen - -

-

- Frontend: v{frontendVersion} - {backendVersion && ( - <> - {" | "} - Backend: v{backendVersion} - - )} - {renderEncodingWorkerStatus()} - {renderFlacfetchStatus()} - {renderAudioSeparatorStatus()} -

-
-
+ <> +
+
+

+ Powered by{" "} + + karaoke-gen + +

+

setModalOpen(true)} + title="Click for detailed system status" + > + Frontend: v{frontendVersion} + {backendVersion && ( + <> + {" | "} + Backend: v{backendVersion} + + )} + {renderEncodingWorkerStatus()} + {renderFlacfetchStatus()} + {renderAudioSeparatorStatus()} +

+
+
+ + setModalOpen(false)} + /> + ); } diff --git a/frontend/lib/api.ts b/frontend/lib/api.ts index a63164c0..88469582 100644 --- a/frontend/lib/api.ts +++ b/frontend/lib/api.ts @@ -304,6 +304,43 @@ export interface AudioSeparatorHealth { error?: string; } +export interface ServiceStatus { + status: string; // 'ok' | 'offline' + version?: string | null; + deployed_at?: string | null; + commit_sha?: string | null; + pr_number?: string | null; + pr_title?: string | null; + active_jobs?: number; + admin_details?: { + // Encoder blue-green + primary_vm?: string; + primary_ip?: string; + primary_version?: string; + primary_deployed_at?: string; + secondary_vm?: string; + secondary_ip?: string; + secondary_version?: string; + secondary_deployed_at?: string; + last_swap_at?: string; + deploy_in_progress?: boolean; + active_jobs?: number; + queue_length?: number; + // Error details for offline services + error?: string | null; + }; +} + +export interface SystemStatus { + services: { + frontend: ServiceStatus; + backend: ServiceStatus; + encoder: ServiceStatus; + flacfetch: ServiceStatus; + separator: ServiceStatus; + }; +} + // Signed URL upload types export interface SignedUploadUrl { file_type: string; @@ -1322,6 +1359,16 @@ export const api = { return handleResponse(response); }, + /** + * Get aggregated system status for all services + */ + async getSystemStatus(): Promise { + const response = await fetch(`${API_BASE_URL}/api/health/system-status`, { + headers: getAuthHeaders(), + }); + return handleResponse(response); + }, + // ========================================================================== // Credits/Payment API endpoints // ========================================================================== diff --git a/frontend/next.config.mjs b/frontend/next.config.mjs index 1e558ccc..780b5aca 100644 --- a/frontend/next.config.mjs +++ b/frontend/next.config.mjs @@ -20,6 +20,10 @@ try { const nextConfig = { env: { NEXT_PUBLIC_APP_VERSION: appVersion, + NEXT_PUBLIC_COMMIT_SHA: process.env.NEXT_PUBLIC_COMMIT_SHA || '', + NEXT_PUBLIC_PR_NUMBER: process.env.NEXT_PUBLIC_PR_NUMBER || '', + NEXT_PUBLIC_PR_TITLE: process.env.NEXT_PUBLIC_PR_TITLE || '', + NEXT_PUBLIC_BUILD_TIME: process.env.NEXT_PUBLIC_BUILD_TIME || new Date().toISOString(), }, // Only use static export for production builds (not dev) output: process.env.NODE_ENV === 'production' ? 'export' : undefined, diff --git a/pyproject.toml b/pyproject.toml index 05be4c98..d09e0f4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.155.3" +version = "0.156.0" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"