From fb9d86d17ff12142de7c0ae3850237cd5a2ff4e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andrew=20Beveridge=20=E2=98=84=EF=B8=8F?= Date: Thu, 17 Sep 2026 13:57:57 -0400 Subject: [PATCH] fix(e2e): hide test-runner jobs from admin, stop stranding + name drift (#1019) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to the investigation of why E2E "piri/dog" jobs started appearing in the live admin dashboard. Three related fixes (the transcription-cache cost bug from the same investigation ships separately in #1017). ## #2 — Hide `e2e-test-runner` jobs from the admin "My Jobs" view `list_jobs`' `exclude_test` filter uses `is_test_email()`, which only matched `@inbox.testmail.app`. When the CI post-deploy canary + daily E2E workflows switched to impersonating the **persistent** `e2e-test-runner@nomadkaraoke.com` account, their jobs stopped being filtered and leaked into the admin dashboard (previously every E2E job ran under an ephemeral testmail user and was hidden). Fix: add a `TEST_EMAIL_ADDRESSES` allowlist in `backend/utils/test_data.py` so this internal test account is treated as test data everywhere `is_test_email()` is used. ## #3 — Safety-net teardown for the happy-path spec `happy-path-real-user.spec.ts` only deleted its job at **Step 12**, which runs solely on a fully successful pass. A run that failed/timed out earlier (e.g. stuck at `awaiting_review`) stranded its job in production forever. Added a `test.afterEach` that deletes the tracked job on every attempt (including retries), regardless of where the test stopped. Idempotent with Step 12. ## #5 — Deterministic E2E song identity (`piri` vs `piri & tommy`) The guided-flow **AI match-judge** nondeterministically rewrote the typed artist to its canonical form (`piri` → `piri & tommy`) depending on the audio-confidence tier and LLM output — a legitimate product feature, but it produced two differently-named jobs across runs. The test now stubs `POST /api/catalog/match-judge` to a no-op verdict echoing the typed values, pinning the job to `piri - dog`. **Product behavior is unchanged** — only the test is pinned. ## Testing - New `backend/tests/test_test_data_utils.py` — `is_test_email` / `is_internal_email` classification incl. `e2e-test-runner` (case-insensitive), plus real/team accounts. - Extended the existing `exclude_test` summary-dict test to include the runner account. - Frontend spec typechecks (`tsc`) and lints (`eslint`) clean. Co-authored-by: Claude Opus 4.8 --- backend/tests/test_routes_jobs.py | 3 + backend/tests/test_test_data_utils.py | 52 ++++++++++++++ backend/utils/test_data.py | 18 ++++- .../production/happy-path-real-user.spec.ts | 68 +++++++++++++++++++ pyproject.toml | 2 +- 5 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 backend/tests/test_test_data_utils.py diff --git a/backend/tests/test_routes_jobs.py b/backend/tests/test_routes_jobs.py index eaa197dc..18f3f7fd 100644 --- a/backend/tests/test_routes_jobs.py +++ b/backend/tests/test_routes_jobs.py @@ -770,6 +770,9 @@ class TestSummaryEndpoint: {'job_id': '1', 'user_email': 'real@example.com'}, {'job_id': '2', 'user_email': 'test@inbox.testmail.app'}, {'job_id': '3', 'user_email': None}, + # The persistent account impersonated by the CI canary + daily E2E + # runs. Its jobs must be filtered out of the admin dashboard too. + {'job_id': '4', 'user_email': 'e2e-test-runner@nomadkaraoke.com'}, ] filtered = [j for j in job_dicts if not is_test_email(j.get('user_email') or "")] diff --git a/backend/tests/test_test_data_utils.py b/backend/tests/test_test_data_utils.py new file mode 100644 index 00000000..6fd28abd --- /dev/null +++ b/backend/tests/test_test_data_utils.py @@ -0,0 +1,52 @@ +"""Unit tests for backend.utils.test_data email classification helpers.""" + +from backend.utils.test_data import ( + is_test_email, + is_internal_email, + is_internal_or_test_email, +) + + +class TestIsTestEmail: + def test_testmail_domain_is_test(self): + assert is_test_email("k1l43.test-123@inbox.testmail.app") is True + + def test_e2e_test_runner_account_is_test(self): + # Persistent account impersonated by the CI canary + daily E2E runs. + assert is_test_email("e2e-test-runner@nomadkaraoke.com") is True + + def test_e2e_test_runner_is_case_insensitive(self): + assert is_test_email("E2E-Test-Runner@NomadKaraoke.com") is True + + def test_regular_team_account_is_not_test(self): + # Other @nomadkaraoke.com accounts are internal, not test data. + assert is_test_email("andrew@nomadkaraoke.com") is False + + def test_real_user_is_not_test(self): + assert is_test_email("real@example.com") is False + + def test_empty_is_not_test(self): + assert is_test_email("") is False + assert is_test_email(None) is False + + +class TestIsInternalEmail: + def test_nomadkaraoke_is_internal(self): + assert is_internal_email("andrew@nomadkaraoke.com") is True + + def test_e2e_runner_is_also_internal_domain(self): + # It lives on the internal domain; is_internal stays True. The point of + # the TEST_EMAIL_ADDRESSES entry is that is_test_email ALSO returns True + # so the admin dashboard's exclude_test filter hides it. + assert is_internal_email("e2e-test-runner@nomadkaraoke.com") is True + + def test_real_user_is_not_internal(self): + assert is_internal_email("real@example.com") is False + + +class TestIsInternalOrTestEmail: + def test_e2e_runner_matches(self): + assert is_internal_or_test_email("e2e-test-runner@nomadkaraoke.com") is True + + def test_real_user_does_not_match(self): + assert is_internal_or_test_email("real@example.com") is False diff --git a/backend/utils/test_data.py b/backend/utils/test_data.py index 0eb48278..572f5c5e 100644 --- a/backend/utils/test_data.py +++ b/backend/utils/test_data.py @@ -10,7 +10,18 @@ abuse investigation views. # Email domains used by automated testing frameworks TEST_EMAIL_DOMAINS = [ - "inbox.testmail.app", # Used by E2E happy path tests + "inbox.testmail.app", # Used by E2E happy path tests (ephemeral throwaway users) +] + +# Specific internal accounts that are only ever used by automated tests. +# These live on the internal nomadkaraoke.com domain (so they'd otherwise pass +# is_internal_email, not is_test_email), but every job they create is E2E test +# data. Listed here so admin dashboards' exclude_test filter hides their jobs. +# e2e-test-runner is the persistent account the CI post-deploy canary and daily +# E2E workflows impersonate; without this entry its piri/dog jobs leak into the +# admin "My Jobs" view. +TEST_EMAIL_ADDRESSES = [ + "e2e-test-runner@nomadkaraoke.com", ] # Email domains belonging to internal team members @@ -28,11 +39,14 @@ def is_test_email(email: str) -> bool: email: Email address to check Returns: - True if the email matches a test email domain pattern + True if the email matches a test email domain pattern or is a known + automated-test account """ if not email: return False email_lower = email.lower() + if email_lower in TEST_EMAIL_ADDRESSES: + return True return any(email_lower.endswith(f"@{domain}") for domain in TEST_EMAIL_DOMAINS) diff --git a/frontend/e2e/production/happy-path-real-user.spec.ts b/frontend/e2e/production/happy-path-real-user.spec.ts index 5dc862a9..85cba401 100644 --- a/frontend/e2e/production/happy-path-real-user.spec.ts +++ b/frontend/e2e/production/happy-path-real-user.spec.ts @@ -1,6 +1,7 @@ import { test, expect, Page, BrowserContext } from '@playwright/test'; import { createEmailHelper, isEmailTestingAvailable } from '../helpers/email-testing'; import { clickCompleteSignInGate } from '../helpers/auth'; +import { deleteTestJob } from '../helpers/test-cleanup'; /** * E2E Happy Path Test - Real User Journey with Full UI Interactions @@ -221,6 +222,31 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => { // retries, so there's no steady-state cost. test.describe.configure({ retries: 1 }); + // Safety-net job tracker. The in-flow cleanup (Step 12) only runs on a fully + // successful pass through review -> complete. If the test fails or times out + // before Step 12 (e.g. stuck at "awaiting_review"), the job it created was + // being stranded in production under the e2e-test-runner account — visible in + // the admin dashboard and never cleaned up. The afterEach below deletes it + // regardless of where the test stopped, on every attempt (incl. retries). + let capturedJobId: string | null = null; + + test.afterEach(async () => { + if (!capturedJobId) return; + const jobToDelete = capturedJobId; + capturedJobId = null; + const adminToken = process.env.E2E_ADMIN_TOKEN; + if (!adminToken) { + console.log(` [teardown] No E2E_ADMIN_TOKEN — cannot delete job ${jobToDelete}; manual cleanup needed`); + return; + } + try { + const result = await deleteTestJob(jobToDelete, adminToken, { cleanupDistribution: true }, API_URL); + console.log(` [teardown] ${result.message}`); + } catch (e) { + console.log(` [teardown] Failed to delete job ${jobToDelete}: ${e}`); + } + }); + test('Complete flow: New user signup -> Karaoke generation -> Distribution -> Cleanup', async ({ page, context, @@ -260,6 +286,46 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => { } }); + // Pin the song identity to the typed input so the created job is + // deterministically "piri - dog". The AI match-judge normally runs on the + // guided-flow search and may rewrite the typed artist to its canonical form + // (e.g. "piri" -> "piri & tommy"), nondeterministically depending on the + // audio-confidence tier and LLM output. That's a legitimate product feature, + // but it made the daily/canary jobs show up under two different names. Here + // we stub the judge to a no-op "none" verdict echoing the typed values so + // the correction never applies. (Does not affect the generation pipeline the + // happy path exists to exercise.) + await page.route('**/api/catalog/match-judge', async (route) => { + const request = route.request(); + if (request.method() === 'POST') { + let artist = TEST_SONG.artist; + let title = TEST_SONG.title; + try { + const body = request.postDataJSON(); + if (body?.artist) artist = body.artist; + if (body?.title) title = body.title; + } catch { + // fall back to TEST_SONG + } + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + kind: 'none', + confident: true, + canonical_artist: artist, + canonical_title: title, + alternatives: [], + engine: 'deterministic', + reason: 'E2E: match-judge stubbed to keep song identity deterministic', + needs_ai: false, + }), + }); + } else { + await route.continue(); + } + }); + try { // ========================================================================= // STEP 1: Landing Page @@ -591,6 +657,7 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => { const idMatch = idText.match(/ID:\s*([a-f0-9]{8,})/i); if (idMatch) { jobId = idMatch[1]; + capturedJobId = jobId; // hand to afterEach so a partial run still cleans up console.log(` Job ID: ${jobId}`); } } @@ -1308,6 +1375,7 @@ test.describe('E2E Happy Path - Real User with Full UI Interactions', () => { console.log(` Dropbox: ${result.dropbox?.status}`); console.log(` GDrive: ${result.gdrive?.status}`); console.log(` Job deleted: ${result.job_deleted}`); + if (result.job_deleted) capturedJobId = null; // already gone; skip afterEach re-delete } else { console.log(` WARNING: Cleanup failed with status ${cleanupResponse.status()}`); } diff --git a/pyproject.toml b/pyproject.toml index a6eaf3f2..838ebf68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "karaoke-gen" -version = "0.228.3" +version = "0.228.5" description = "Generate karaoke videos with synchronized lyrics. Handles the entire process from downloading audio and lyrics to creating the final video with title screens." authors = ["Andrew Beveridge "] license = "MIT"