mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
feat(requests-board): admin history view + fix community publish transition (#1008)
## Why Andrew asked whether we can see past community-requests-board submissions and whether our first real user got a video. Investigation surfaced: - **First real user was fully served.** `tas041964@gmail.com` requested *Sam Cooke and the Soul Stirrers – Wonderful* (Sep 15); it was auto-picked, generated, published to YouTube (NOMAD-1650), and he got the completion email + emailed thanks. ✅ - **No admin view existed** to see this — you had to query Firestore directly. - **A real bug:** the community publish fan-out (`mark_published` + voter emails) fired **only** on the `youtube_upload_queue` path. Jobs published via the direct video-worker distribution path (like Sam Cooke's) bypassed it, so the request stayed `in_progress`, never lit up the board's public "Recently made" list, and other up-voters would silently miss their "track is live" email. ## What 1. **Fix the publish transition** — extract the fan-out into `backend/services/community_publish.py` and call it from **both** publish paths (`youtube_queue_processor` + `video_worker`). Also fixes a latent `len(int)` crash in the log line. 2. **Backfill / safety net** — `POST /api/internal/community-requests/reconcile`: idempotent, publishes any `in_progress` pick whose job is already live on YouTube. (Will be run in prod post-deploy to fix the Sam Cooke request.) 3. **Admin history view** — `GET /api/admin/community-requests` + a `/admin/community-requests` page: every request, all statuses, with requester, votes, and job + YouTube links. The public board's "Recently made" section already existed in the UI — it was empty only because of bug #1; fixing the transition (+ reconcile backfill) lights it up. ## Testing - `backend/tests/test_community_voter_notify.py` — fan-out from the shared module, both-path idempotency, voter dedupe/retry, YouTube-url resolution, and the reconcile scan. - `backend/tests/test_community_review_admin.py` — the new admin history endpoint maps all statuses. - All 31 requests-board + 158 worker tests pass. The one unrelated failure in `test_audio_processor_local_gpu.py` is pre-existing (confirmed with changes stashed). ## Notes - Admin surface is English-only (no i18n changes); no `en.json` strings added. - Local CodeRabbit review couldn't run (CLI requires life360-org SSO; no seat on this repo) — leaving to the GitHub bot. Plan: `docs/archive/2026-09-17-requests-board-history-and-publish-transition-plan.md` 🤖 Generated with [Claude Code](https://claude.com/claude-code) <!-- This is an auto-generated comment: release notes by coderabbit.ai --> ## Summary by CodeRabbit - **New Features** - Added an admin Community Requests history page covering all statuses, votes, submitters, review details, linked jobs, and YouTube videos. - Added admin reconciliation for eligible requests with live videos. - Added automatic voter email notifications after community-request publication. - **Bug Fixes** - Improved publication handling across direct and queued video workflows. - Added recovery for incomplete voter notifications and missed publication transitions. - **Documentation** - Documented the community-request history and reconciliation API endpoints. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
1 parent
083a478aba
commit
36b331f3d7
14 files changed
+705
-70
No files matched your search
@@ -4202,6 +4202,52 @@ def list_user_feedback(
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Requests board — full request history (admin view)
|
||||
# =============================================================================
|
||||
|
||||
class CommunityRequestItem(BaseModel):
|
||||
"""A requests-board song request, any status, for the admin history view."""
|
||||
id: str
|
||||
artist: str
|
||||
title: str
|
||||
status: str
|
||||
submitted_by: str
|
||||
owner_email: Optional[str] = None
|
||||
vote_count: int
|
||||
job_id: Optional[str] = None
|
||||
youtube_url: Optional[str] = None
|
||||
review_state: Optional[str] = None
|
||||
created_at: Optional[str] = None
|
||||
picked_at: Optional[str] = None
|
||||
|
||||
|
||||
class CommunityRequestListResponse(BaseModel):
|
||||
requests: List[CommunityRequestItem]
|
||||
|
||||
|
||||
@router.get("/community-requests", response_model=CommunityRequestListResponse)
|
||||
async def list_community_requests(
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin),
|
||||
):
|
||||
"""Full history of requests-board submissions across every status — who asked,
|
||||
how it was voted, which job made it, and where it landed on YouTube."""
|
||||
from backend.services.song_request_service import get_song_request_service
|
||||
service = get_song_request_service()
|
||||
items = [
|
||||
CommunityRequestItem(
|
||||
id=r.id, artist=r.artist, title=r.title, status=r.status,
|
||||
submitted_by=r.submitted_by, owner_email=r.owner_email,
|
||||
vote_count=r.vote_count, job_id=r.job_id, youtube_url=r.youtube_url,
|
||||
review_state=r.review_state,
|
||||
created_at=r.created_at.isoformat() if r.created_at else None,
|
||||
picked_at=r.picked_at.isoformat() if r.picked_at else None,
|
||||
)
|
||||
for r in service.list_all()
|
||||
]
|
||||
return CommunityRequestListResponse(requests=items)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Requests board — existing-community-version review queue
|
||||
# =============================================================================
|
||||
|
||||
@@ -684,6 +684,30 @@ async def community_daily_pick_endpoint(
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/community-requests/reconcile")
|
||||
async def community_requests_reconcile_endpoint(
|
||||
http_request: Request,
|
||||
auth_data: Tuple[str, UserType, int] = Depends(require_admin)
|
||||
):
|
||||
"""Reconcile community picks whose video is already live on YouTube but whose
|
||||
request is still ``in_progress`` (publish fan-out never fired for them).
|
||||
|
||||
Idempotent safety net / backfill: advances each to ``published`` and fans out
|
||||
voter emails. Safe to call any time.
|
||||
"""
|
||||
from backend.services.community_publish import reconcile_community_publishes
|
||||
|
||||
logger.info("COMMUNITY_REQUESTS_RECONCILE starting")
|
||||
add_span_attribute("operation", "community_requests_reconcile")
|
||||
try:
|
||||
result = await reconcile_community_publishes()
|
||||
logger.info(f"COMMUNITY_REQUESTS_RECONCILE complete: {result}")
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.exception(f"COMMUNITY_REQUESTS_RECONCILE failed: {e}")
|
||||
raise HTTPException(status_code=500, detail=str(e))
|
||||
|
||||
|
||||
@router.post("/community-handoffs")
|
||||
async def community_handoffs_endpoint(
|
||||
http_request: Request,
|
||||
|
||||
@@ -0,0 +1,154 @@
|
||||
"""Requests-board publish fan-out.
|
||||
|
||||
When a community-pick job goes live on YouTube, its ``song_request`` must advance
|
||||
to ``published`` and every up-voter (other than the owner, who already got the job
|
||||
completion email) should get a "your track is live" email.
|
||||
|
||||
A job can reach YouTube via **two** paths, and both must trigger this:
|
||||
1. the quota-managed ``youtube_upload_queue`` (``youtube_queue_processor``), and
|
||||
2. a direct upload during video-worker distribution (``video_worker``), which
|
||||
never touches that queue.
|
||||
|
||||
Historically only path (1) ran the fan-out, so direct-published community picks
|
||||
(e.g. the first real user request) stayed stuck at ``in_progress`` and their
|
||||
board "Recently made" entry never appeared. This shared helper is called from
|
||||
both paths so the outcome is identical regardless of how the video was published.
|
||||
"""
|
||||
import logging
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _youtube_url_for_job(job) -> Optional[str]:
|
||||
"""The published YouTube URL for a job, if any, checking both the state_data
|
||||
field (set on the direct-publish path) and the distribution metadata."""
|
||||
if job is None:
|
||||
return None
|
||||
state = getattr(job, "state_data", None) or {}
|
||||
url = state.get("youtube_url")
|
||||
if url:
|
||||
return url
|
||||
meta = getattr(job, "processing_metadata", None) or {}
|
||||
dist = meta.get("distribution") or {}
|
||||
return dist.get("youtube_video_url") or None
|
||||
|
||||
|
||||
async def notify_community_publish(job_id: str, youtube_url: str) -> Optional[str]:
|
||||
"""Advance a community pick to ``published`` and fan out voter emails.
|
||||
|
||||
Idempotent and best-effort: safe to call from both publish paths and to
|
||||
re-run after a partial failure. Marking published is always safe to repeat;
|
||||
voter emails are guarded by the request's ``notified_voters`` /
|
||||
``voters_notified`` flags so no one is emailed twice.
|
||||
|
||||
Returns the request id if this was a community pick, else ``None`` (a normal,
|
||||
non-community job — no-op).
|
||||
"""
|
||||
try:
|
||||
from backend.services.song_request_service import get_song_request_service
|
||||
service = get_song_request_service()
|
||||
request = service.get_by_job_id(job_id)
|
||||
if request is None:
|
||||
return None # Not a community pick — normal job.
|
||||
|
||||
service.mark_published(request.id, youtube_url)
|
||||
|
||||
if request.voters_notified:
|
||||
return request.id
|
||||
|
||||
# Email up-voters we haven't already reached (retry-safe): exclude the
|
||||
# owner (already got the completion email) and anyone previously notified.
|
||||
owner = (request.owner_email or request.submitted_by or "").lower()
|
||||
already = {v.lower() for v in (request.notified_voters or [])}
|
||||
pending = [
|
||||
v for v in service.list_upvoters(request.id)
|
||||
if v != owner and v not in already
|
||||
]
|
||||
|
||||
from backend.services.job_notification_service import get_job_notification_service
|
||||
notification_service = get_job_notification_service()
|
||||
succeeded = 0
|
||||
for voter in pending:
|
||||
try:
|
||||
ok = await notification_service.send_community_track_live_email(
|
||||
to_email=voter,
|
||||
artist=request.artist,
|
||||
title=request.title,
|
||||
youtube_url=youtube_url,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed community voter email for job %s / %s", job_id, voter)
|
||||
continue
|
||||
if ok:
|
||||
# Record each success immediately so a crash mid-loop never
|
||||
# re-emails an already-notified voter on the retry.
|
||||
service.add_notified_voters(request.id, [voter])
|
||||
succeeded += 1
|
||||
|
||||
# Only flag "fully notified" when every pending voter was reached — a
|
||||
# partial failure leaves the flag unset so a re-run retries just the misses.
|
||||
if succeeded == len(pending):
|
||||
service.mark_voters_notified(request.id)
|
||||
logger.info(
|
||||
"community pick %s published (job %s): notified %d/%d pending voters",
|
||||
request.id, job_id, succeeded, len(pending),
|
||||
)
|
||||
return request.id
|
||||
except Exception as e:
|
||||
logger.error("Failed community voter fan-out for job %s: %s", job_id, e)
|
||||
return None
|
||||
|
||||
|
||||
async def reconcile_community_publishes() -> Dict[str, Any]:
|
||||
"""Safety net / backfill for the community publish fan-out. Two passes:
|
||||
|
||||
1. Picks stuck at ``in_progress`` whose job is actually live on YouTube — run
|
||||
the publish transition (covers picks published before the fan-out was wired
|
||||
into every path, or any future path that forgets to call it).
|
||||
2. Picks already ``published`` but with ``voters_notified == False`` — a voter
|
||||
send failed (or raised) after the request was marked published, and no other
|
||||
path retries them. Re-run the fan-out (idempotent; only un-notified voters
|
||||
are re-emailed).
|
||||
|
||||
Fully idempotent — safe to call any time.
|
||||
"""
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.song_request_service import get_song_request_service
|
||||
|
||||
service = get_song_request_service()
|
||||
job_manager = JobManager()
|
||||
|
||||
scanned = 0
|
||||
published = []
|
||||
|
||||
# Pass 1: in_progress picks whose job is live.
|
||||
for request in service.list_in_progress():
|
||||
scanned += 1
|
||||
if not request.job_id:
|
||||
continue
|
||||
url = _youtube_url_for_job(job_manager.get_job(request.job_id))
|
||||
if not url:
|
||||
continue
|
||||
result_id = await notify_community_publish(request.job_id, url)
|
||||
if result_id:
|
||||
published.append({"request_id": result_id, "job_id": request.job_id, "youtube_url": url})
|
||||
|
||||
# Pass 2: published picks whose voter fan-out never fully completed.
|
||||
retried = 0
|
||||
for request in service.list_published_unnotified():
|
||||
scanned += 1
|
||||
if not request.job_id:
|
||||
continue
|
||||
# Prefer the URL already recorded on the request; fall back to the job.
|
||||
url = request.youtube_url or _youtube_url_for_job(job_manager.get_job(request.job_id))
|
||||
if not url:
|
||||
continue
|
||||
if await notify_community_publish(request.job_id, url):
|
||||
retried += 1
|
||||
|
||||
logger.info(
|
||||
"community publish reconcile: scanned %d, published %d, retried-fanout %d",
|
||||
scanned, len(published), retried,
|
||||
)
|
||||
return {"scanned": scanned, "published": published, "retried_fanout": retried}
|
||||
@@ -170,6 +170,15 @@ class SongRequestService:
|
||||
return SongRequest(**doc.to_dict())
|
||||
return None
|
||||
|
||||
def list_all(self) -> list[SongRequest]:
|
||||
"""Every request across all statuses, newest first — the admin history view."""
|
||||
items = [
|
||||
SongRequest(**doc.to_dict())
|
||||
for doc in self.db.collection(REQUESTS_COLLECTION).stream()
|
||||
]
|
||||
items.sort(key=lambda r: r.created_at, reverse=True)
|
||||
return items
|
||||
|
||||
def list_active(self) -> list[SongRequest]:
|
||||
"""Open requests, ranked by net votes desc then oldest-first."""
|
||||
query = self.db.collection(REQUESTS_COLLECTION).where(
|
||||
@@ -388,6 +397,19 @@ class SongRequestService:
|
||||
).limit(ACTIVE_FETCH_LIMIT)
|
||||
return [SongRequest(**doc.to_dict()) for doc in query.stream()]
|
||||
|
||||
def list_published_unnotified(self) -> list[SongRequest]:
|
||||
"""Published picks whose voter fan-out never fully completed
|
||||
(``voters_notified == False``) — e.g. a send failed after the request was
|
||||
already marked published. The reconcile job retries these; the per-voter
|
||||
``notified_voters`` guard keeps it from re-emailing anyone already reached.
|
||||
"""
|
||||
query = self.db.collection(REQUESTS_COLLECTION).where(
|
||||
filter=FieldFilter("status", "==", "published")
|
||||
).where(
|
||||
filter=FieldFilter("voters_notified", "==", False)
|
||||
).limit(ACTIVE_FETCH_LIMIT)
|
||||
return [SongRequest(**doc.to_dict()) for doc in query.stream()]
|
||||
|
||||
def list_upvoters(self, request_id: str) -> list[str]:
|
||||
"""Emails that up-voted this request (value > 0), oldest vote first.
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ from fastapi import HTTPException
|
||||
from backend.api.routes.admin import (
|
||||
CommunityReviewActionBody,
|
||||
action_community_review,
|
||||
list_community_requests,
|
||||
)
|
||||
from backend.models.song_request import SongRequest
|
||||
|
||||
@@ -105,3 +106,26 @@ async def test_make_failure_raises_500():
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await _call(_patches(svc, provision=provision), "req1", "make")
|
||||
assert exc.value.status_code == 500
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_community_requests_maps_all_statuses():
|
||||
svc = MagicMock()
|
||||
published = _req(
|
||||
id="pub", status="published", review_state=None,
|
||||
job_id="j-abc12345", youtube_url="https://youtu.be/pub",
|
||||
owner_email="voter@x.com",
|
||||
)
|
||||
open_req = _req(id="op", status="open", review_state=None, job_id=None, youtube_url=None)
|
||||
svc.list_all.return_value = [published, open_req]
|
||||
with patch("backend.services.song_request_service.get_song_request_service",
|
||||
MagicMock(return_value=svc)):
|
||||
res = await list_community_requests(auth_data=("admin", None, 0))
|
||||
assert [r.id for r in res.requests] == ["pub", "op"]
|
||||
pub = res.requests[0]
|
||||
assert pub.status == "published"
|
||||
assert pub.youtube_url == "https://youtu.be/pub"
|
||||
assert pub.owner_email == "voter@x.com"
|
||||
assert pub.created_at is not None # datetime serialized to iso string
|
||||
# Open request carries no job/video links.
|
||||
assert res.requests[1].job_id is None and res.requests[1].youtube_url is None
|
||||
@@ -1,10 +1,14 @@
|
||||
"""Unit tests for the requests-board publish fan-out in the YouTube queue processor."""
|
||||
"""Unit tests for the requests-board publish fan-out (backend.services.community_publish).
|
||||
|
||||
The fan-out fires from BOTH publish paths (youtube_queue_processor and the direct
|
||||
video_worker distribution), so it lives in a shared module and is tested here once.
|
||||
"""
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.models.song_request import SongRequest
|
||||
from backend.workers import youtube_queue_processor as yqp
|
||||
from backend.services import community_publish as cp
|
||||
|
||||
|
||||
def _req(**over):
|
||||
@@ -34,7 +38,8 @@ async def test_non_community_job_is_ignored():
|
||||
notifier = MagicMock()
|
||||
p1, p2 = _ctx(svc, notifier)
|
||||
with p1, p2:
|
||||
await yqp._notify_community_voters("j1", {}, "https://youtu.be/x")
|
||||
result = await cp.notify_community_publish("j1", "https://youtu.be/x")
|
||||
assert result is None
|
||||
svc.mark_published.assert_not_called()
|
||||
|
||||
|
||||
@@ -47,7 +52,8 @@ async def test_publishes_and_fans_out_excluding_owner():
|
||||
notifier.send_community_track_live_email = AsyncMock(return_value=True)
|
||||
p1, p2 = _ctx(svc, notifier)
|
||||
with p1, p2:
|
||||
await yqp._notify_community_voters("j1", {}, "https://youtu.be/x")
|
||||
result = await cp.notify_community_publish("j1", "https://youtu.be/x")
|
||||
assert result == "req1"
|
||||
svc.mark_published.assert_called_once_with("req1", "https://youtu.be/x")
|
||||
# Owner excluded (already got the completion email); two voters emailed.
|
||||
emailed = {c.kwargs["to_email"] for c in notifier.send_community_track_live_email.call_args_list}
|
||||
@@ -69,7 +75,7 @@ async def test_skips_already_notified_voters_and_retries_failures():
|
||||
notifier.send_community_track_live_email = AsyncMock(return_value=False) # v3 send fails
|
||||
p1, p2 = _ctx(svc, notifier)
|
||||
with p1, p2:
|
||||
await yqp._notify_community_voters("j1", {}, "https://youtu.be/x")
|
||||
await cp.notify_community_publish("j1", "https://youtu.be/x")
|
||||
# Only the un-notified voter (v3) is attempted; v2 skipped.
|
||||
emailed = {c.kwargs["to_email"] for c in notifier.send_community_track_live_email.call_args_list}
|
||||
assert emailed == {"v3@x.com"}
|
||||
@@ -85,7 +91,97 @@ async def test_already_notified_does_not_resend():
|
||||
notifier.send_community_track_live_email = AsyncMock(return_value=True)
|
||||
p1, p2 = _ctx(svc, notifier)
|
||||
with p1, p2:
|
||||
await yqp._notify_community_voters("j1", {}, "https://youtu.be/x")
|
||||
await cp.notify_community_publish("j1", "https://youtu.be/x")
|
||||
svc.mark_published.assert_called_once() # republish (mark) is safe/idempotent
|
||||
notifier.send_community_track_live_email.assert_not_called()
|
||||
svc.mark_voters_notified.assert_not_called()
|
||||
|
||||
|
||||
# --- youtube-url resolution ---------------------------------------------------
|
||||
|
||||
def test_youtube_url_prefers_state_data():
|
||||
job = MagicMock()
|
||||
job.state_data = {"youtube_url": "https://youtu.be/state"}
|
||||
job.processing_metadata = {"distribution": {"youtube_video_url": "https://youtu.be/dist"}}
|
||||
assert cp._youtube_url_for_job(job) == "https://youtu.be/state"
|
||||
|
||||
|
||||
def test_youtube_url_falls_back_to_distribution_metadata():
|
||||
job = MagicMock()
|
||||
job.state_data = {}
|
||||
job.processing_metadata = {"distribution": {"youtube_video_url": "https://youtu.be/dist"}}
|
||||
assert cp._youtube_url_for_job(job) == "https://youtu.be/dist"
|
||||
|
||||
|
||||
def test_youtube_url_none_when_unpublished():
|
||||
job = MagicMock()
|
||||
job.state_data = {}
|
||||
job.processing_metadata = {"distribution": {}}
|
||||
assert cp._youtube_url_for_job(job) is None
|
||||
assert cp._youtube_url_for_job(None) is None
|
||||
|
||||
|
||||
# --- reconcile / backfill -----------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconcile_publishes_live_in_progress_requests():
|
||||
svc = MagicMock()
|
||||
live = _req(id="r-live", job_id="j-live", voters_notified=True)
|
||||
unpub = _req(id="r-unpub", job_id="j-unpub", voters_notified=True)
|
||||
nojob = _req(id="r-nojob", job_id=None)
|
||||
svc.list_in_progress.return_value = [live, unpub, nojob]
|
||||
svc.list_published_unnotified.return_value = []
|
||||
# get_by_job_id (used inside notify_community_publish) resolves the live request.
|
||||
svc.get_by_job_id.side_effect = lambda jid: {"j-live": live}.get(jid)
|
||||
|
||||
live_job = MagicMock(state_data={"youtube_url": "https://youtu.be/live"}, processing_metadata={})
|
||||
unpub_job = MagicMock(state_data={}, processing_metadata={"distribution": {}})
|
||||
|
||||
jm = MagicMock()
|
||||
jm.get_job.side_effect = lambda jid: {"j-live": live_job, "j-unpub": unpub_job}.get(jid)
|
||||
|
||||
with patch.multiple(
|
||||
"backend.services.song_request_service",
|
||||
get_song_request_service=MagicMock(return_value=svc),
|
||||
), patch("backend.services.job_manager.JobManager", MagicMock(return_value=jm)):
|
||||
result = await cp.reconcile_community_publishes()
|
||||
|
||||
assert result["scanned"] == 3
|
||||
published_ids = {p["request_id"] for p in result["published"]}
|
||||
assert published_ids == {"r-live"}
|
||||
assert result["retried_fanout"] == 0
|
||||
svc.mark_published.assert_called_once_with("r-live", "https://youtu.be/live")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reconcile_retries_published_but_unnotified_fanout():
|
||||
# A published pick whose voter fan-out never completed is retried using the
|
||||
# URL already on the request (no job lookup needed).
|
||||
svc = MagicMock()
|
||||
svc.list_in_progress.return_value = []
|
||||
stuck = _req(
|
||||
id="r-stuck", job_id="j-stuck", status="published",
|
||||
youtube_url="https://youtu.be/stuck", voters_notified=False,
|
||||
)
|
||||
svc.list_published_unnotified.return_value = [stuck]
|
||||
svc.get_by_job_id.side_effect = lambda jid: {"j-stuck": stuck}.get(jid)
|
||||
svc.list_upvoters.return_value = ["owner@x.com", "v2@x.com"]
|
||||
notifier = MagicMock()
|
||||
notifier.send_community_track_live_email = AsyncMock(return_value=True)
|
||||
|
||||
jm = MagicMock()
|
||||
jm.get_job.return_value = None # not needed — request already carries the URL
|
||||
|
||||
with patch.multiple(
|
||||
"backend.services.song_request_service",
|
||||
get_song_request_service=MagicMock(return_value=svc),
|
||||
), patch(
|
||||
"backend.services.job_notification_service.get_job_notification_service",
|
||||
MagicMock(return_value=notifier),
|
||||
), patch("backend.services.job_manager.JobManager", MagicMock(return_value=jm)):
|
||||
result = await cp.reconcile_community_publishes()
|
||||
|
||||
assert result["retried_fanout"] == 1
|
||||
# The un-notified voter (not the owner) gets emailed on the retry.
|
||||
emailed = {c.kwargs["to_email"] for c in notifier.send_community_track_live_email.call_args_list}
|
||||
assert emailed == {"v2@x.com"}
|
||||
@@ -432,6 +432,18 @@ async def generate_video_orchestrated(job_id: str) -> bool:
|
||||
timeline_metadata=completion_metadata,
|
||||
)
|
||||
|
||||
# If this was a requests-board community pick published directly here
|
||||
# (i.e. not deferred to the youtube_upload_queue), advance its request
|
||||
# to `published` and fan out "your track is live" voter emails. The
|
||||
# queue path fires the same helper; jobs deferred to the queue are
|
||||
# handled there instead, so we skip them to avoid a premature publish.
|
||||
if result.youtube_url and not result.youtube_upload_queued:
|
||||
try:
|
||||
from backend.services.community_publish import notify_community_publish
|
||||
await notify_community_publish(job_id, result.youtube_url)
|
||||
except Exception:
|
||||
logger.exception(f"[job:{job_id}] Community publish fan-out failed (non-fatal)")
|
||||
|
||||
root_span.set_attribute("duration_seconds", duration)
|
||||
root_span.set_attribute("brand_code", result.brand_code or '')
|
||||
logger.info(f"[job:{job_id}] WORKER_END worker=video orchestrator=true status=success duration={duration:.1f}s")
|
||||
@@ -1024,6 +1036,15 @@ async def generate_video_legacy(job_id: str) -> bool:
|
||||
timeline_metadata=completion_metadata,
|
||||
)
|
||||
|
||||
# Advance a requests-board community pick to `published` + fan out
|
||||
# voter emails (legacy path always publishes directly here).
|
||||
if result.get('youtube_url'):
|
||||
try:
|
||||
from backend.services.community_publish import notify_community_publish
|
||||
await notify_community_publish(job_id, result.get('youtube_url'))
|
||||
except Exception:
|
||||
logger.exception(f"[job:{job_id}] Community publish fan-out failed (non-fatal)")
|
||||
|
||||
root_span.set_attribute("duration_seconds", duration)
|
||||
root_span.set_attribute("brand_code", result.get('brand_code', ''))
|
||||
logger.info(f"[job:{job_id}] WORKER_END worker=video status=success duration={duration:.1f}s")
|
||||
|
||||
@@ -11,6 +11,7 @@ import tempfile
|
||||
from typing import Dict, Any, Optional
|
||||
|
||||
from backend.config import get_settings
|
||||
from backend.services.community_publish import notify_community_publish
|
||||
from backend.services.job_manager import JobManager
|
||||
from backend.services.storage_service import StorageService
|
||||
from backend.services.youtube_quota_service import get_youtube_quota_service
|
||||
@@ -91,7 +92,7 @@ async def process_youtube_upload_queue() -> Dict[str, Any]:
|
||||
|
||||
# If this was a requests-board community pick, mark it published
|
||||
# and fan out "your track is live" emails to everyone who voted.
|
||||
await _notify_community_voters(job_id, entry, youtube_url)
|
||||
await notify_community_publish(job_id, youtube_url)
|
||||
|
||||
processed += 1
|
||||
else:
|
||||
@@ -304,68 +305,6 @@ def _update_job_youtube_url(job_id: str, youtube_url: str) -> None:
|
||||
logger.error(f"Failed to update job {job_id} with YouTube URL: {e}")
|
||||
|
||||
|
||||
async def _notify_community_voters(
|
||||
job_id: str, entry: Dict[str, Any], youtube_url: str
|
||||
) -> None:
|
||||
"""Publish fan-out for requests-board picks: mark the request published and
|
||||
email every up-voter (except the owner, who already got the completion email).
|
||||
|
||||
Idempotent via the request's ``voters_notified`` flag. Best-effort — never
|
||||
fails the upload over a notification error.
|
||||
"""
|
||||
try:
|
||||
from backend.services.song_request_service import get_song_request_service
|
||||
service = get_song_request_service()
|
||||
request = service.get_by_job_id(job_id)
|
||||
if request is None:
|
||||
return # Not a community pick — normal job.
|
||||
|
||||
service.mark_published(request.id, youtube_url)
|
||||
|
||||
if request.voters_notified:
|
||||
return
|
||||
|
||||
# Email up-voters we haven't already reached (retry-safe): exclude the
|
||||
# owner (already got the completion email) and anyone previously notified.
|
||||
owner = (request.owner_email or request.submitted_by or "").lower()
|
||||
already = {v.lower() for v in (request.notified_voters or [])}
|
||||
pending = [
|
||||
v for v in service.list_upvoters(request.id)
|
||||
if v != owner and v not in already
|
||||
]
|
||||
|
||||
from backend.services.job_notification_service import get_job_notification_service
|
||||
notification_service = get_job_notification_service()
|
||||
succeeded = 0
|
||||
for voter in pending:
|
||||
try:
|
||||
ok = await notification_service.send_community_track_live_email(
|
||||
to_email=voter,
|
||||
artist=request.artist,
|
||||
title=request.title,
|
||||
youtube_url=youtube_url,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("Failed community voter email for job %s / %s", job_id, voter)
|
||||
continue
|
||||
if ok:
|
||||
# Record each success immediately so a crash mid-loop never
|
||||
# re-emails an already-notified voter on the retry.
|
||||
service.add_notified_voters(request.id, [voter])
|
||||
succeeded += 1
|
||||
|
||||
# Only flag "fully notified" when every pending voter was reached — a
|
||||
# partial failure leaves the flag unset so a re-run retries just the misses.
|
||||
if succeeded == len(pending):
|
||||
service.mark_voters_notified(request.id)
|
||||
logger.info(
|
||||
"community pick %s published (job %s): notified %d/%d pending voters",
|
||||
request.id, job_id, len(succeeded), len(pending),
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed community voter fan-out for job {job_id}: {e}")
|
||||
|
||||
|
||||
async def _send_youtube_upload_notification(
|
||||
job_id: str, entry: Dict[str, Any], youtube_url: str
|
||||
) -> None:
|
||||
|
||||
+18
@@ -1448,6 +1448,24 @@ X-Admin-Token: ADMIN_TOKEN
|
||||
|
||||
Internal (admin/OIDC). Called hourly by Cloud Scheduler. For each in-progress community pick whose owner hasn't completed their review within 24h (and whose job is blocked waiting on the owner), reassigns the job to the next up-voter (oldest-first, skipping those already tried) and emails them — up to 5 voters, then parks the track (`stalled`). Returns `{status, checked, handed_off, parked, errors}`.
|
||||
|
||||
### Community Publish Reconcile (internal)
|
||||
|
||||
```http
|
||||
POST /api/internal/community-requests/reconcile
|
||||
X-Admin-Token: ADMIN_TOKEN
|
||||
```
|
||||
|
||||
Internal (admin/OIDC). Idempotent safety net / backfill. A community pick advances to `published` (and fans out "your track is live" emails to up-voters) when its job goes live on YouTube — this now fires from both publish paths (the `youtube_upload_queue` processor and the direct video-worker distribution). This endpoint reconciles any pick still stuck at `in_progress` whose job actually has a YouTube URL, running the publish transition for each. Returns `{scanned, published: [{request_id, job_id, youtube_url}]}`.
|
||||
|
||||
### Community Requests History (admin)
|
||||
|
||||
```http
|
||||
GET /api/admin/community-requests
|
||||
X-Admin-Token: ADMIN_TOKEN
|
||||
```
|
||||
|
||||
Admin. Full history of requests-board submissions across every status (newest first), surfaced at `/admin/community-requests`. Each item: `{id, artist, title, status, submitted_by, owner_email, vote_count, job_id, youtube_url, review_state, created_at, picked_at}` — who asked, how it was voted, which job made it, and where it landed on YouTube.
|
||||
|
||||
### Community Review Queue (existing-version, admin)
|
||||
|
||||
```http
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
# Requests Board — Admin History View, Public "Recently Made" List, & Publish-Transition Fix
|
||||
|
||||
Date: 2026-09-17
|
||||
Branch: `feat/sess-20260916-2353-requests-board-history`
|
||||
|
||||
## Context / discovery
|
||||
|
||||
Andrew asked whether we can see past community requests and whether our first real user
|
||||
successfully got a video. Investigation (Firestore + `email_log`):
|
||||
|
||||
- **Only 2 requests ever.** Both stuck at `status == "in_progress"`:
|
||||
- Admin go-live test request — Sep 3 (submitted by `admin@`, job since deleted).
|
||||
- A real community user's request — **Sep 15** (first genuine board user; PII kept out of this
|
||||
public repo — see the private session record / Firestore for specifics).
|
||||
- **First real user was fully served.** The job completed, rendered all finals, and published to
|
||||
YouTube (brand assigned, GDrive + Dropbox). `email_log` confirms magic-link → `action_reminder`
|
||||
(lyrics review) → `job_completion` emails all sent to the requester; he emailed a thank-you. ✅
|
||||
(He was notified via the **standard job-owner** path, not the community fan-out.)
|
||||
|
||||
### Root-cause bug found
|
||||
The community publish transition (`song_request_service.mark_published` + voter fan-out in
|
||||
`_notify_community_voters`) is invoked **only** inside `backend/workers/youtube_queue_processor.py`
|
||||
(the quota-managed `youtube_upload_queue` path). The first real user's job got its YouTube URL via
|
||||
the **direct distribution path** during finalization (`processing_metadata.distribution.youtube_video_url`),
|
||||
which never enqueues to `youtube_upload_queue`. Result: `_notify_community_voters` never ran, so the
|
||||
request stayed `in_progress`, `youtube_url` stayed empty, `voters_notified` stayed `False`.
|
||||
|
||||
Impact today is minor (he was the only voter, and got the standard completion email anyway), but:
|
||||
- The board's public **"Recently made"** section (`frontend/app/[locale]/requests/client.tsx:317`,
|
||||
driven by `list_published()` = `status == "published"`) will **never populate** while this bug
|
||||
exists — so it looks empty/broken.
|
||||
- Any request with *other* up-voters would silently fail to send the "your track is live" fan-out.
|
||||
|
||||
## Scope (agreed with Andrew)
|
||||
|
||||
1. **Admin history view** of ALL community requests (his explicit ask).
|
||||
2. **Public "recently made" list** on the board — already coded, needs the publish transition fixed
|
||||
+ a backfill to light up.
|
||||
3. **Fix the publish-transition bug** (prerequisite for #2; correctness for all future picks).
|
||||
|
||||
## Plan
|
||||
|
||||
### Part A — Publish transition fix (backend)
|
||||
- Extract the "job just got a YouTube URL → advance its community request" logic into a single
|
||||
reusable helper (e.g. `song_request_service.on_job_published(job_id, youtube_url)` or a shared
|
||||
`_notify_community_voters` callable) that: looks up the request by `state_data.community_request_id`,
|
||||
calls `mark_published`, and fans out voter emails (retry-safe via existing `notified_voters` /
|
||||
`voters_notified` flags).
|
||||
- Call it from **both** publish paths:
|
||||
- existing `youtube_queue_processor` site (line ~94) — unchanged behavior;
|
||||
- the **direct distribution finalize path** — add the hook where `distribution.youtube_video_url`
|
||||
is set. Locate that write and call the shared helper there (idempotent, so double-firing is safe).
|
||||
- Keep it idempotent so re-runs / both-paths-firing don't double-email.
|
||||
|
||||
### Part B — Backfill the first real request
|
||||
- Run the reconcile endpoint (Part below) in prod to mark the stuck request `published` with its
|
||||
YouTube URL, making it appear in the board's "Recently made" list. In general, reconcile also
|
||||
retries missing community "track is live" emails for any un-notified **non-owner** up-voters via
|
||||
`notify_community_publish()`. For *this specific* request no extra email is sent — its sole voter
|
||||
is the owner, who was already notified via the standard job-completion email.
|
||||
|
||||
### Part C — Admin history view (backend + frontend)
|
||||
- **Backend:** `GET /api/admin/community-requests` (admin-auth) returning ALL `song_requests`,
|
||||
every status, with: artist/title, status, `submitted_by`, `owner_email`, `vote_count`, `job_id`,
|
||||
`youtube_url`, `created_at`, `picked_at`, `review_state`. Reuse existing admin auth + response
|
||||
model patterns from `admin.py` (see `community-reviews`).
|
||||
- **Frontend:** new admin page `frontend/app/admin/community-requests/` (English-only admin surface;
|
||||
no `[locale]` counterpart) — a table with status badges, requester, link to the job review page
|
||||
(`/app/jobs#/{job_id}/review`), and YouTube link. Add nav entry alongside `community-reviews`.
|
||||
|
||||
### Testing
|
||||
- Backend unit: shared helper fires from the distribution path; idempotency; request lookup by
|
||||
`state_data.community_request_id`; admin endpoint returns all statuses + is admin-gated.
|
||||
- Frontend: admin page renders rows, links resolve.
|
||||
- Follow `docs/TESTING.md`; run `make test`.
|
||||
|
||||
## Open questions
|
||||
- Exact location of the direct-distribution YouTube-URL write (need to grep the finalize/distribute
|
||||
worker) — confirm during implementation.
|
||||
- Whether to also surface the admin history inside the existing `/admin/community-reviews` page vs a
|
||||
new page. Leaning new page for clarity.
|
||||
@@ -0,0 +1,179 @@
|
||||
"use client"
|
||||
|
||||
import { useCallback, useEffect, useMemo, useState } from "react"
|
||||
import { adminApi } from "@/lib/api"
|
||||
import type { CommunityRequestItem } from "@/lib/api"
|
||||
import { Card, CardContent } from "@/components/ui/card"
|
||||
import { Button } from "@/components/ui/button"
|
||||
import { Badge } from "@/components/ui/badge"
|
||||
import { Skeleton } from "@/components/ui/skeleton"
|
||||
import {
|
||||
Table, TableHeader, TableBody, TableHead, TableRow, TableCell,
|
||||
} from "@/components/ui/table"
|
||||
import { History, Youtube, ExternalLink, RefreshCw } from "lucide-react"
|
||||
|
||||
// Status → badge styling. Mirrors the request lifecycle in backend/models/song_request.py.
|
||||
const STATUS_VARIANT: Record<string, "default" | "secondary" | "outline" | "destructive"> = {
|
||||
open: "secondary",
|
||||
queued: "outline",
|
||||
in_progress: "outline",
|
||||
published: "default",
|
||||
rejected: "destructive",
|
||||
stalled: "destructive",
|
||||
}
|
||||
|
||||
function fmtDate(iso?: string | null): string {
|
||||
if (!iso) return "—"
|
||||
const d = new Date(iso)
|
||||
return isNaN(d.getTime()) ? "—" : d.toLocaleDateString(undefined, { year: "numeric", month: "short", day: "numeric" })
|
||||
}
|
||||
|
||||
export default function CommunityRequestsPage() {
|
||||
const [requests, setRequests] = useState<CommunityRequestItem[]>([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [error, setError] = useState<string | null>(null)
|
||||
|
||||
const load = useCallback(async () => {
|
||||
setLoading(true)
|
||||
setError(null)
|
||||
try {
|
||||
const res = await adminApi.listCommunityRequests()
|
||||
setRequests(res.requests)
|
||||
} catch {
|
||||
setError("Failed to load community requests.")
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
load()
|
||||
}, [load])
|
||||
|
||||
const counts = useMemo(() => {
|
||||
const c: Record<string, number> = {}
|
||||
for (const r of requests) c[r.status] = (c[r.status] ?? 0) + 1
|
||||
return c
|
||||
}, [requests])
|
||||
|
||||
return (
|
||||
<div className="space-y-6 max-w-6xl">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold flex items-center gap-2">
|
||||
<History className="w-6 h-6" /> Community Requests
|
||||
</h1>
|
||||
<p className="text-sm text-muted-foreground mt-1">
|
||||
Every song requested via the public voting board
|
||||
(<a href="https://requests.nomadkaraoke.com" target="_blank" rel="noopener noreferrer"
|
||||
className="text-blue-500 hover:underline">requests.nomadkaraoke.com</a>) —
|
||||
who asked, how it was voted, which job made it, and where it landed on YouTube.
|
||||
</p>
|
||||
</div>
|
||||
<Button variant="outline" size="sm" onClick={load} disabled={loading}>
|
||||
<RefreshCw className={`w-4 h-4 mr-1 ${loading ? "animate-spin" : ""}`} /> Refresh
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="rounded-lg px-3 py-2 text-sm bg-red-500/10 text-red-500 border border-red-500/30">
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{!loading && requests.length > 0 && (
|
||||
<div className="flex flex-wrap gap-2 text-xs">
|
||||
<Badge variant="outline">{requests.length} total</Badge>
|
||||
{Object.entries(counts).map(([status, n]) => (
|
||||
<Badge key={status} variant={STATUS_VARIANT[status] ?? "outline"}>
|
||||
{n} {status.replace("_", " ")}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{loading ? (
|
||||
<div className="space-y-3">
|
||||
{[0, 1, 2].map((i) => <Skeleton key={i} className="h-12 w-full" />)}
|
||||
</div>
|
||||
) : requests.length === 0 ? (
|
||||
<Card>
|
||||
<CardContent className="py-10 text-center text-muted-foreground">
|
||||
No community requests yet.
|
||||
</CardContent>
|
||||
</Card>
|
||||
) : (
|
||||
<Card>
|
||||
<CardContent className="p-0">
|
||||
<Table>
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Song</TableHead>
|
||||
<TableHead>Status</TableHead>
|
||||
<TableHead>Requester</TableHead>
|
||||
<TableHead className="text-right">Votes</TableHead>
|
||||
<TableHead>Requested</TableHead>
|
||||
<TableHead>Job</TableHead>
|
||||
<TableHead>Video</TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{requests.map((r) => (
|
||||
<TableRow key={r.id}>
|
||||
<TableCell className="font-medium">
|
||||
{r.artist} — {r.title}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Badge variant={STATUS_VARIANT[r.status] ?? "outline"}>
|
||||
{r.status.replace("_", " ")}
|
||||
</Badge>
|
||||
{r.review_state ? (
|
||||
<Badge variant="outline" className="ml-1">{r.review_state}</Badge>
|
||||
) : null}
|
||||
</TableCell>
|
||||
<TableCell className="text-sm">
|
||||
<span className="truncate">{r.submitted_by}</span>
|
||||
{r.owner_email && r.owner_email !== r.submitted_by ? (
|
||||
<span className="block text-xs text-muted-foreground">owner: {r.owner_email}</span>
|
||||
) : null}
|
||||
</TableCell>
|
||||
<TableCell className="text-right">{r.vote_count}</TableCell>
|
||||
<TableCell className="text-sm text-muted-foreground">{fmtDate(r.created_at)}</TableCell>
|
||||
<TableCell>
|
||||
{r.job_id ? (
|
||||
<a
|
||||
href={`/app/jobs#/${r.job_id}`}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="inline-flex items-center gap-1 text-sm text-blue-500 hover:underline"
|
||||
>
|
||||
{r.job_id.slice(0, 8)} <ExternalLink className="w-3 h-3" />
|
||||
</a>
|
||||
) : (
|
||||
<span className="text-muted-foreground text-sm">—</span>
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{r.youtube_url ? (
|
||||
<a
|
||||
href={r.youtube_url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
className="inline-flex items-center gap-1 text-sm text-blue-500 hover:underline"
|
||||
>
|
||||
<Youtube className="w-4 h-4 text-red-500" /> Watch
|
||||
</a>
|
||||
) : (
|
||||
<span className="text-muted-foreground text-sm">—</span>
|
||||
)}
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -18,6 +18,7 @@ import {
|
||||
Share2,
|
||||
Building2,
|
||||
ListChecks,
|
||||
History,
|
||||
} from "lucide-react"
|
||||
|
||||
import {
|
||||
@@ -60,6 +61,11 @@ const navItems = [
|
||||
href: "/admin/referrals",
|
||||
icon: Share2,
|
||||
},
|
||||
{
|
||||
title: "Community Requests",
|
||||
href: "/admin/community-requests",
|
||||
icon: History,
|
||||
},
|
||||
{
|
||||
title: "Community Reviews",
|
||||
href: "/admin/community-reviews",
|
||||
|
||||
@@ -2617,7 +2617,32 @@ export interface CommunityReviewItem {
|
||||
community_checked_at?: string | null;
|
||||
}
|
||||
|
||||
export interface CommunityRequestItem {
|
||||
id: string;
|
||||
artist: string;
|
||||
title: string;
|
||||
status: string;
|
||||
submitted_by: string;
|
||||
owner_email?: string | null;
|
||||
vote_count: number;
|
||||
job_id?: string | null;
|
||||
youtube_url?: string | null;
|
||||
review_state?: string | null;
|
||||
created_at?: string | null;
|
||||
picked_at?: string | null;
|
||||
}
|
||||
|
||||
export const adminApi = {
|
||||
/**
|
||||
* Full history of requests-board submissions across every status.
|
||||
*/
|
||||
async listCommunityRequests(): Promise<{ requests: CommunityRequestItem[] }> {
|
||||
const response = await apiFetch(`${API_BASE_URL}/api/admin/community-requests`, {
|
||||
headers: getAuthHeaders(),
|
||||
});
|
||||
return handleResponse(response);
|
||||
},
|
||||
|
||||
/**
|
||||
* List requests-board picks flagged for existing-community-version review.
|
||||
*/
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.225.10"
|
||||
version = "0.226.0"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user