mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
fix(e2e): hide test-runner jobs from admin, stop stranding + name drift (#1019)
Follow-up to the investigation of why E2E "piri/dog" jobs started appearing in the live admin dashboard. Three related fixes (the transcription-cache cost bug from the same investigation ships separately in #1017). ## #2 — Hide `e2e-test-runner` jobs from the admin "My Jobs" view `list_jobs`' `exclude_test` filter uses `is_test_email()`, which only matched `@inbox.testmail.app`. When the CI post-deploy canary + daily E2E workflows switched to impersonating the **persistent** `e2e-test-runner@nomadkaraoke.com` account, their jobs stopped being filtered and leaked into the admin dashboard (previously every E2E job ran under an ephemeral testmail user and was hidden). Fix: add a `TEST_EMAIL_ADDRESSES` allowlist in `backend/utils/test_data.py` so this internal test account is treated as test data everywhere `is_test_email()` is used. ## #3 — Safety-net teardown for the happy-path spec `happy-path-real-user.spec.ts` only deleted its job at **Step 12**, which runs solely on a fully successful pass. A run that failed/timed out earlier (e.g. stuck at `awaiting_review`) stranded its job in production forever. Added a `test.afterEach` that deletes the tracked job on every attempt (including retries), regardless of where the test stopped. Idempotent with Step 12. ## #5 — Deterministic E2E song identity (`piri` vs `piri & tommy`) The guided-flow **AI match-judge** nondeterministically rewrote the typed artist to its canonical form (`piri` → `piri & tommy`) depending on the audio-confidence tier and LLM output — a legitimate product feature, but it produced two differently-named jobs across runs. The test now stubs `POST /api/catalog/match-judge` to a no-op verdict echoing the typed values, pinning the job to `piri - dog`. **Product behavior is unchanged** — only the test is pinned. ## Testing - New `backend/tests/test_test_data_utils.py` — `is_test_email` / `is_internal_email` classification incl. `e2e-test-runner` (case-insensitive), plus real/team accounts. - Extended the existing `exclude_test` summary-dict test to include the runner account. - Frontend spec typechecks (`tsc`) and lints (`eslint`) clean. Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
1 parent
11bf84f296
commit
fb9d86d17f
5 files changed
+140
-3
No files matched your search
@@ -770,6 +770,9 @@ class TestSummaryEndpoint:
|
||||
{'job_id': '1', 'user_email': 'real@example.com'},
|
||||
{'job_id': '2', 'user_email': 'test@inbox.testmail.app'},
|
||||
{'job_id': '3', 'user_email': None},
|
||||
# The persistent account impersonated by the CI canary + daily E2E
|
||||
# runs. Its jobs must be filtered out of the admin dashboard too.
|
||||
{'job_id': '4', 'user_email': 'e2e-test-runner@nomadkaraoke.com'},
|
||||
]
|
||||
|
||||
filtered = [j for j in job_dicts if not is_test_email(j.get('user_email') or "")]
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Unit tests for backend.utils.test_data email classification helpers."""
|
||||
|
||||
from backend.utils.test_data import (
|
||||
is_test_email,
|
||||
is_internal_email,
|
||||
is_internal_or_test_email,
|
||||
)
|
||||
|
||||
|
||||
class TestIsTestEmail:
|
||||
def test_testmail_domain_is_test(self):
|
||||
assert is_test_email("k1l43.test-123@inbox.testmail.app") is True
|
||||
|
||||
def test_e2e_test_runner_account_is_test(self):
|
||||
# Persistent account impersonated by the CI canary + daily E2E runs.
|
||||
assert is_test_email("e2e-test-runner@nomadkaraoke.com") is True
|
||||
|
||||
def test_e2e_test_runner_is_case_insensitive(self):
|
||||
assert is_test_email("E2E-Test-Runner@NomadKaraoke.com") is True
|
||||
|
||||
def test_regular_team_account_is_not_test(self):
|
||||
# Other @nomadkaraoke.com accounts are internal, not test data.
|
||||
assert is_test_email("andrew@nomadkaraoke.com") is False
|
||||
|
||||
def test_real_user_is_not_test(self):
|
||||
assert is_test_email("real@example.com") is False
|
||||
|
||||
def test_empty_is_not_test(self):
|
||||
assert is_test_email("") is False
|
||||
assert is_test_email(None) is False
|
||||
|
||||
|
||||
class TestIsInternalEmail:
|
||||
def test_nomadkaraoke_is_internal(self):
|
||||
assert is_internal_email("andrew@nomadkaraoke.com") is True
|
||||
|
||||
def test_e2e_runner_is_also_internal_domain(self):
|
||||
# It lives on the internal domain; is_internal stays True. The point of
|
||||
# the TEST_EMAIL_ADDRESSES entry is that is_test_email ALSO returns True
|
||||
# so the admin dashboard's exclude_test filter hides it.
|
||||
assert is_internal_email("e2e-test-runner@nomadkaraoke.com") is True
|
||||
|
||||
def test_real_user_is_not_internal(self):
|
||||
assert is_internal_email("real@example.com") is False
|
||||
|
||||
|
||||
class TestIsInternalOrTestEmail:
|
||||
def test_e2e_runner_matches(self):
|
||||
assert is_internal_or_test_email("e2e-test-runner@nomadkaraoke.com") is True
|
||||
|
||||
def test_real_user_does_not_match(self):
|
||||
assert is_internal_or_test_email("real@example.com") is False
|
||||
@@ -10,7 +10,18 @@ abuse investigation views.
|
||||
|
||||
# Email domains used by automated testing frameworks
|
||||
TEST_EMAIL_DOMAINS = [
|
||||
"inbox.testmail.app", # Used by E2E happy path tests
|
||||
"inbox.testmail.app", # Used by E2E happy path tests (ephemeral throwaway users)
|
||||
]
|
||||
|
||||
# Specific internal accounts that are only ever used by automated tests.
|
||||
# These live on the internal nomadkaraoke.com domain (so they'd otherwise pass
|
||||
# is_internal_email, not is_test_email), but every job they create is E2E test
|
||||
# data. Listed here so admin dashboards' exclude_test filter hides their jobs.
|
||||
# e2e-test-runner is the persistent account the CI post-deploy canary and daily
|
||||
# E2E workflows impersonate; without this entry its piri/dog jobs leak into the
|
||||
# admin "My Jobs" view.
|
||||
TEST_EMAIL_ADDRESSES = [
|
||||
"e2e-test-runner@nomadkaraoke.com",
|
||||
]
|
||||
|
||||
# Email domains belonging to internal team members
|
||||
@@ -28,11 +39,14 @@ def is_test_email(email: str) -> bool:
|
||||
email: Email address to check
|
||||
|
||||
Returns:
|
||||
True if the email matches a test email domain pattern
|
||||
True if the email matches a test email domain pattern or is a known
|
||||
automated-test account
|
||||
"""
|
||||
if not email:
|
||||
return False
|
||||
email_lower = email.lower()
|
||||
if email_lower in TEST_EMAIL_ADDRESSES:
|
||||
return True
|
||||
return any(email_lower.endswith(f"@{domain}") for domain in TEST_EMAIL_DOMAINS)
|
||||
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { test, expect, Page, BrowserContext } from '@playwright/test';
|
||||
import { createEmailHelper, isEmailTestingAvailable } from '../helpers/email-testing';
|
||||
import { clickCompleteSignInGate } from '../helpers/auth';
|
||||
import { deleteTestJob } from '../helpers/test-cleanup';
|
||||
|
||||
/**
|
||||
* E2E Happy Path Test - Real User Journey with Full UI Interactions
|
||||
@@ -221,6 +222,31 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
|
||||
// retries, so there's no steady-state cost.
|
||||
test.describe.configure({ retries: 1 });
|
||||
|
||||
// Safety-net job tracker. The in-flow cleanup (Step 12) only runs on a fully
|
||||
// successful pass through review -> complete. If the test fails or times out
|
||||
// before Step 12 (e.g. stuck at "awaiting_review"), the job it created was
|
||||
// being stranded in production under the e2e-test-runner account — visible in
|
||||
// the admin dashboard and never cleaned up. The afterEach below deletes it
|
||||
// regardless of where the test stopped, on every attempt (incl. retries).
|
||||
let capturedJobId: string | null = null;
|
||||
|
||||
test.afterEach(async () => {
|
||||
if (!capturedJobId) return;
|
||||
const jobToDelete = capturedJobId;
|
||||
capturedJobId = null;
|
||||
const adminToken = process.env.E2E_ADMIN_TOKEN;
|
||||
if (!adminToken) {
|
||||
console.log(` [teardown] No E2E_ADMIN_TOKEN — cannot delete job ${jobToDelete}; manual cleanup needed`);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const result = await deleteTestJob(jobToDelete, adminToken, { cleanupDistribution: true }, API_URL);
|
||||
console.log(` [teardown] ${result.message}`);
|
||||
} catch (e) {
|
||||
console.log(` [teardown] Failed to delete job ${jobToDelete}: ${e}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('Complete flow: New user signup -> Karaoke generation -> Distribution -> Cleanup', async ({
|
||||
page,
|
||||
context,
|
||||
@@ -260,6 +286,46 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
|
||||
}
|
||||
});
|
||||
|
||||
// Pin the song identity to the typed input so the created job is
|
||||
// deterministically "piri - dog". The AI match-judge normally runs on the
|
||||
// guided-flow search and may rewrite the typed artist to its canonical form
|
||||
// (e.g. "piri" -> "piri & tommy"), nondeterministically depending on the
|
||||
// audio-confidence tier and LLM output. That's a legitimate product feature,
|
||||
// but it made the daily/canary jobs show up under two different names. Here
|
||||
// we stub the judge to a no-op "none" verdict echoing the typed values so
|
||||
// the correction never applies. (Does not affect the generation pipeline the
|
||||
// happy path exists to exercise.)
|
||||
await page.route('**/api/catalog/match-judge', async (route) => {
|
||||
const request = route.request();
|
||||
if (request.method() === 'POST') {
|
||||
let artist = TEST_SONG.artist;
|
||||
let title = TEST_SONG.title;
|
||||
try {
|
||||
const body = request.postDataJSON();
|
||||
if (body?.artist) artist = body.artist;
|
||||
if (body?.title) title = body.title;
|
||||
} catch {
|
||||
// fall back to TEST_SONG
|
||||
}
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({
|
||||
kind: 'none',
|
||||
confident: true,
|
||||
canonical_artist: artist,
|
||||
canonical_title: title,
|
||||
alternatives: [],
|
||||
engine: 'deterministic',
|
||||
reason: 'E2E: match-judge stubbed to keep song identity deterministic',
|
||||
needs_ai: false,
|
||||
}),
|
||||
});
|
||||
} else {
|
||||
await route.continue();
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
// =========================================================================
|
||||
// STEP 1: Landing Page
|
||||
@@ -591,6 +657,7 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
|
||||
const idMatch = idText.match(/ID:\s*([a-f0-9]{8,})/i);
|
||||
if (idMatch) {
|
||||
jobId = idMatch[1];
|
||||
capturedJobId = jobId; // hand to afterEach so a partial run still cleans up
|
||||
console.log(` Job ID: ${jobId}`);
|
||||
}
|
||||
}
|
||||
@@ -1308,6 +1375,7 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
|
||||
console.log(` Dropbox: ${result.dropbox?.status}`);
|
||||
console.log(` GDrive: ${result.gdrive?.status}`);
|
||||
console.log(` Job deleted: ${result.job_deleted}`);
|
||||
if (result.job_deleted) capturedJobId = null; // already gone; skip afterEach re-delete
|
||||
} else {
|
||||
console.log(` WARNING: Cleanup failed with status ${cleanupResponse.status()}`);
|
||||
}
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "karaoke-gen"
|
||||
version = "0.228.3"
|
||||
version = "0.228.5"
|
||||
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
|
||||
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
|
||||
license = "MIT"
|
||||
|
||||
Reference in new issue
Block a user