fix(e2e): hide test-runner jobs from admin, stop stranding + name drift (#1019)

Follow-up to the investigation of why E2E "piri/dog" jobs started
appearing in the live admin dashboard. Three related fixes (the
transcription-cache cost bug from the same investigation ships
separately in #1017).

## #2 — Hide `e2e-test-runner` jobs from the admin "My Jobs" view
`list_jobs`' `exclude_test` filter uses `is_test_email()`, which only
matched `@inbox.testmail.app`. When the CI post-deploy canary + daily
E2E workflows switched to impersonating the **persistent**
`e2e-test-runner@nomadkaraoke.com` account, their jobs stopped being
filtered and leaked into the admin dashboard (previously every E2E job
ran under an ephemeral testmail user and was hidden).

Fix: add a `TEST_EMAIL_ADDRESSES` allowlist in
`backend/utils/test_data.py` so this internal test account is treated as
test data everywhere `is_test_email()` is used.

## #3 — Safety-net teardown for the happy-path spec
`happy-path-real-user.spec.ts` only deleted its job at **Step 12**,
which runs solely on a fully successful pass. A run that failed/timed
out earlier (e.g. stuck at `awaiting_review`) stranded its job in
production forever. Added a `test.afterEach` that deletes the tracked
job on every attempt (including retries), regardless of where the test
stopped. Idempotent with Step 12.

## #5 — Deterministic E2E song identity (`piri` vs `piri & tommy`)
The guided-flow **AI match-judge** nondeterministically rewrote the
typed artist to its canonical form (`piri` → `piri & tommy`) depending
on the audio-confidence tier and LLM output — a legitimate product
feature, but it produced two differently-named jobs across runs. The
test now stubs `POST /api/catalog/match-judge` to a no-op verdict
echoing the typed values, pinning the job to `piri - dog`. **Product
behavior is unchanged** — only the test is pinned.

## Testing
- New `backend/tests/test_test_data_utils.py` — `is_test_email` /
`is_internal_email` classification incl. `e2e-test-runner`
(case-insensitive), plus real/team accounts.
- Extended the existing `exclude_test` summary-dict test to include the
runner account.
- Frontend spec typechecks (`tsc`) and lints (`eslint`) clean.

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Andrew Beveridge ☄️andClaude Opus 4.8 authored and GitHub committed 2026-09-17 13:57:57 -04:00
1 parent 11bf84f296
commit fb9d86d17f
5 files changed
+140 -3

No files matched your search

+3
View File
@@ -770,6 +770,9 @@ class TestSummaryEndpoint:
{'job_id': '1', 'user_email': 'real@example.com'},
{'job_id': '2', 'user_email': 'test@inbox.testmail.app'},
{'job_id': '3', 'user_email': None},
# The persistent account impersonated by the CI canary + daily E2E
# runs. Its jobs must be filtered out of the admin dashboard too.
{'job_id': '4', 'user_email': 'e2e-test-runner@nomadkaraoke.com'},
]
filtered = [j for j in job_dicts if not is_test_email(j.get('user_email') or "")]
+52
View File
@@ -0,0 +1,52 @@
"""Unit tests for backend.utils.test_data email classification helpers."""
from backend.utils.test_data import (
is_test_email,
is_internal_email,
is_internal_or_test_email,
)
class TestIsTestEmail:
def test_testmail_domain_is_test(self):
assert is_test_email("k1l43.test-123@inbox.testmail.app") is True
def test_e2e_test_runner_account_is_test(self):
# Persistent account impersonated by the CI canary + daily E2E runs.
assert is_test_email("e2e-test-runner@nomadkaraoke.com") is True
def test_e2e_test_runner_is_case_insensitive(self):
assert is_test_email("E2E-Test-Runner@NomadKaraoke.com") is True
def test_regular_team_account_is_not_test(self):
# Other @nomadkaraoke.com accounts are internal, not test data.
assert is_test_email("andrew@nomadkaraoke.com") is False
def test_real_user_is_not_test(self):
assert is_test_email("real@example.com") is False
def test_empty_is_not_test(self):
assert is_test_email("") is False
assert is_test_email(None) is False
class TestIsInternalEmail:
def test_nomadkaraoke_is_internal(self):
assert is_internal_email("andrew@nomadkaraoke.com") is True
def test_e2e_runner_is_also_internal_domain(self):
# It lives on the internal domain; is_internal stays True. The point of
# the TEST_EMAIL_ADDRESSES entry is that is_test_email ALSO returns True
# so the admin dashboard's exclude_test filter hides it.
assert is_internal_email("e2e-test-runner@nomadkaraoke.com") is True
def test_real_user_is_not_internal(self):
assert is_internal_email("real@example.com") is False
class TestIsInternalOrTestEmail:
def test_e2e_runner_matches(self):
assert is_internal_or_test_email("e2e-test-runner@nomadkaraoke.com") is True
def test_real_user_does_not_match(self):
assert is_internal_or_test_email("real@example.com") is False
+16 -2
View File
@@ -10,7 +10,18 @@ abuse investigation views.
# Email domains used by automated testing frameworks
TEST_EMAIL_DOMAINS = [
"inbox.testmail.app", # Used by E2E happy path tests
"inbox.testmail.app", # Used by E2E happy path tests (ephemeral throwaway users)
]
# Specific internal accounts that are only ever used by automated tests.
# These live on the internal nomadkaraoke.com domain (so they'd otherwise pass
# is_internal_email, not is_test_email), but every job they create is E2E test
# data. Listed here so admin dashboards' exclude_test filter hides their jobs.
# e2e-test-runner is the persistent account the CI post-deploy canary and daily
# E2E workflows impersonate; without this entry its piri/dog jobs leak into the
# admin "My Jobs" view.
TEST_EMAIL_ADDRESSES = [
"e2e-test-runner@nomadkaraoke.com",
]
# Email domains belonging to internal team members
@@ -28,11 +39,14 @@ def is_test_email(email: str) -> bool:
email: Email address to check
Returns:
True if the email matches a test email domain pattern
True if the email matches a test email domain pattern or is a known
automated-test account
"""
if not email:
return False
email_lower = email.lower()
if email_lower in TEST_EMAIL_ADDRESSES:
return True
return any(email_lower.endswith(f"@{domain}") for domain in TEST_EMAIL_DOMAINS)
@@ -1,6 +1,7 @@
import { test, expect, Page, BrowserContext } from '@playwright/test';
import { createEmailHelper, isEmailTestingAvailable } from '../helpers/email-testing';
import { clickCompleteSignInGate } from '../helpers/auth';
import { deleteTestJob } from '../helpers/test-cleanup';
/**
* E2E Happy Path Test - Real User Journey with Full UI Interactions
@@ -221,6 +222,31 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
// retries, so there's no steady-state cost.
test.describe.configure({ retries: 1 });
// Safety-net job tracker. The in-flow cleanup (Step 12) only runs on a fully
// successful pass through review -> complete. If the test fails or times out
// before Step 12 (e.g. stuck at "awaiting_review"), the job it created was
// being stranded in production under the e2e-test-runner account — visible in
// the admin dashboard and never cleaned up. The afterEach below deletes it
// regardless of where the test stopped, on every attempt (incl. retries).
let capturedJobId: string | null = null;
test.afterEach(async () => {
if (!capturedJobId) return;
const jobToDelete = capturedJobId;
capturedJobId = null;
const adminToken = process.env.E2E_ADMIN_TOKEN;
if (!adminToken) {
console.log(` [teardown] No E2E_ADMIN_TOKEN — cannot delete job ${jobToDelete}; manual cleanup needed`);
return;
}
try {
const result = await deleteTestJob(jobToDelete, adminToken, { cleanupDistribution: true }, API_URL);
console.log(` [teardown] ${result.message}`);
} catch (e) {
console.log(` [teardown] Failed to delete job ${jobToDelete}: ${e}`);
}
});
test('Complete flow: New user signup -> Karaoke generation -> Distribution -> Cleanup', async ({
page,
context,
@@ -260,6 +286,46 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
}
});
// Pin the song identity to the typed input so the created job is
// deterministically "piri - dog". The AI match-judge normally runs on the
// guided-flow search and may rewrite the typed artist to its canonical form
// (e.g. "piri" -> "piri & tommy"), nondeterministically depending on the
// audio-confidence tier and LLM output. That's a legitimate product feature,
// but it made the daily/canary jobs show up under two different names. Here
// we stub the judge to a no-op "none" verdict echoing the typed values so
// the correction never applies. (Does not affect the generation pipeline the
// happy path exists to exercise.)
await page.route('**/api/catalog/match-judge', async (route) => {
const request = route.request();
if (request.method() === 'POST') {
let artist = TEST_SONG.artist;
let title = TEST_SONG.title;
try {
const body = request.postDataJSON();
if (body?.artist) artist = body.artist;
if (body?.title) title = body.title;
} catch {
// fall back to TEST_SONG
}
await route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({
kind: 'none',
confident: true,
canonical_artist: artist,
canonical_title: title,
alternatives: [],
engine: 'deterministic',
reason: 'E2E: match-judge stubbed to keep song identity deterministic',
needs_ai: false,
}),
});
} else {
await route.continue();
}
});
try {
// =========================================================================
// STEP 1: Landing Page
@@ -591,6 +657,7 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
const idMatch = idText.match(/ID:\s*([a-f0-9]{8,})/i);
if (idMatch) {
jobId = idMatch[1];
capturedJobId = jobId; // hand to afterEach so a partial run still cleans up
console.log(` Job ID: ${jobId}`);
}
}
@@ -1308,6 +1375,7 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => {
console.log(` Dropbox: ${result.dropbox?.status}`);
console.log(` GDrive: ${result.gdrive?.status}`);
console.log(` Job deleted: ${result.job_deleted}`);
if (result.job_deleted) capturedJobId = null; // already gone; skip afterEach re-delete
} else {
console.log(` WARNING: Cleanup failed with status ${cleanupResponse.status()}`);
}
+1 -1
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "karaoke-gen"
version = "0.228.3"
version = "0.228.5"
description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens."
authors = ["Andrew Beveridge <andrew@beveridge.uk>"]
license = "MIT"