mirror of
https://github.com/nomadkaraoke/karaoke-gen.git
synced 2026-11-20 13:07:57 +09:00
test(e2e): sanitized page-activity log on tenant E2E failure (#1229)
## Summary Tenant E2E #155 (singa, 2026-10-28) failed with the portal stuck on the `/en/app/` loading spinner for 30s. The backend served tenant config in 168ms, 8 live re-runs were fine, and run #156 got past the same step. The only evidence was a screenshot: CI has no Playwright traces, because traces hold `X-Admin-Token` and the repo's artifacts are public. This adds a public-safe stand-in, `frontend/e2e/helpers/page-activity.ts`. It records: - main-frame navigations; - each request's method and origin/path (no query strings), with status and duration or the failure reason; - console errors/warnings and uncaught page errors, with URLs and token-like values in their text redacted. On failure it writes `page-activity.log` into test-results (the scrubber still checks it) and prints the requests still in flight to the job log. It is wired into `tenant-happy-path.spec.ts`. The approach is documented in docs/TESTING.md, which also records #155 as a one-off flake. ## Testing - New offline spec `e2e/self-healing/page-activity.spec.ts` (3 tests): a fixture page with one hung, one failed and one token-bearing request, plus console and page errors. The self-heal suite runs in CI's frontend-e2e-smoke job. The whole suite passes locally (36 tests). - `playwright test --list` on the production config compiles the tenant spec. - Local CodeRabbit review: 2 findings (redact console text; race on the REQFAIL assertion), both fixed. - Test-only change, so no version bump. Alert: https://discord.com/channels/1198691101943541770/1265517528608669707/1564876714532798465 @coderabbitai ignore 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_012RYLVTiUPKxn3sQPyp3Teh --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
53bc8aa708
commit
19f3c731fb
4 files changed
+216
No files matched your search
@@ -685,6 +685,25 @@ override), the credit-purchase spec disables trace/video/auto-screenshots, and t
|
||||
own screenshots mask every input. When adding a new secret to a production E2E job, add its env
|
||||
var to `SECRET_ENV_VARS` in the scrubber and to the job's scrub step.
|
||||
|
||||
### Debugging CI failures without a trace: `page-activity.log`
|
||||
|
||||
Since CI has no traces, the tenant happy-path spec records a public-safe stand-in with
|
||||
`e2e/helpers/page-activity.ts` (`recordPageActivity(page)` in `beforeEach`,
|
||||
`writeOnFailure(testInfo)` in `afterEach`). On failure it writes `page-activity.log` to the
|
||||
test's `test-results/` dir (uploaded in the artifact) and prints the requests still in flight
|
||||
to the job log as `[page-activity] …` lines. It holds main-frame navigations, each request's
|
||||
method + origin/path (query strings dropped: magic-link tokens and signed-URL signatures) with
|
||||
status and duration or failure reason, console errors/warnings, and uncaught page errors. It
|
||||
holds no headers or bodies. It is plain text, so the scrubber still checks it. Use the same
|
||||
two hooks in any other production spec that needs it. Offline tests:
|
||||
`e2e/self-healing/page-activity.spec.ts`.
|
||||
|
||||
Added after tenant E2E run #155 (2026-10-28). The singa portal sat on `/en/app/`'s initial
|
||||
spinner for 30s and never redirected to the Sign In landing. The backend served tenant config in
|
||||
168ms, 8 live re-runs were fine, and it didn't recur. The suspected cause, a JS chunk stalled on
|
||||
Cloudflare Pages, was unconfirmable without network detail. If it happens again, the log's
|
||||
`IN FLIGHT AT FAILURE` section shows what hung. Until then, treat it as a one-off flake.
|
||||
|
||||
## Concurrent-Load Regression Test (review pages)
|
||||
|
||||
`scripts/load-test-review.py` replays the 2026-09-18 "10 tabs melt the backend"
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
// frontend/e2e/helpers/page-activity.ts
|
||||
import type { Page, TestInfo } from '@playwright/test';
|
||||
import * as fs from 'fs';
|
||||
|
||||
/**
|
||||
* Lightweight, public-safe stand-in for a Playwright trace on production E2E runs.
|
||||
*
|
||||
* Traces hold every request header (X-Admin-Token) and typed value, so CI keeps
|
||||
* them off and scrub-artifacts.mjs deletes them (karaoke-gen is a PUBLIC repo).
|
||||
* That left failures like "page stuck on the loading spinner" with nothing to go
|
||||
* on but a screenshot. This records only what's needed to see what the page was
|
||||
* waiting on — main-frame navigations, each request's method, origin+path (query
|
||||
* strings dropped: magic-link tokens, signed-URL signatures), status, duration or
|
||||
* failure, plus console errors/warnings and uncaught page errors (URLs/tokens in
|
||||
* their text redacted the same way) — and on failure
|
||||
* writes it to `page-activity.log` in the test's output dir and prints the
|
||||
* requests still in flight.
|
||||
*/
|
||||
|
||||
const MAX_ENTRIES = 2000;
|
||||
const MAX_TEXT = 500;
|
||||
|
||||
/** origin + path only — query strings and fragments can carry tokens/signatures. */
|
||||
export function sanitizeUrl(raw: string): string {
|
||||
try {
|
||||
const u = new URL(raw);
|
||||
if (u.protocol === 'data:' || u.protocol === 'blob:') return `${u.protocol}…`;
|
||||
return `${u.origin}${u.pathname}`;
|
||||
} catch {
|
||||
return raw.split(/[?#]/)[0].slice(0, MAX_TEXT);
|
||||
}
|
||||
}
|
||||
|
||||
/** Console/page-error text can embed signed URLs or tokens: strip query strings and token params. */
|
||||
export function redactText(text: string): string {
|
||||
return text
|
||||
.replace(/[a-z][a-z0-9+.-]*:\/\/[^\s"'<>()]+/gi, (url) => sanitizeUrl(url))
|
||||
.replace(/\b((?:access_|id_|refresh_|magic_)?token|key|signature|sig|code|secret|password)(["']?\s*[=:]\s*["']?)[^\s"'&,;]+/gi, '$1$2[redacted]')
|
||||
.replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/g, 'Bearer [redacted]');
|
||||
}
|
||||
|
||||
function clip(text: string): string {
|
||||
const oneLine = redactText(text).replace(/\s+/g, ' ').trim();
|
||||
return oneLine.length > MAX_TEXT ? `${oneLine.slice(0, MAX_TEXT)}…` : oneLine;
|
||||
}
|
||||
|
||||
export interface PageActivityRecorder {
|
||||
/** Activity lines so far (each prefixed with ms since recording started). */
|
||||
lines(): string[];
|
||||
/** Requests started but not finished/failed, as "METHOD url (Nms)". */
|
||||
pending(): string[];
|
||||
/** Writes page-activity.log + prints in-flight requests when the test failed. */
|
||||
writeOnFailure(testInfo: TestInfo): Promise<void>;
|
||||
}
|
||||
|
||||
export function recordPageActivity(page: Page): PageActivityRecorder {
|
||||
const t0 = Date.now();
|
||||
const entries: string[] = [];
|
||||
const inFlight = new Map<object, { label: string; start: number }>();
|
||||
let dropped = 0;
|
||||
|
||||
const log = (line: string) => {
|
||||
if (entries.length >= MAX_ENTRIES) {
|
||||
dropped++;
|
||||
return;
|
||||
}
|
||||
entries.push(`+${String(Date.now() - t0).padStart(6)}ms ${line}`);
|
||||
};
|
||||
|
||||
page.on('framenavigated', (frame) => {
|
||||
if (frame === page.mainFrame()) log(`NAV ${sanitizeUrl(frame.url())}`);
|
||||
});
|
||||
page.on('request', (req) => {
|
||||
inFlight.set(req, { label: `${req.method()} ${sanitizeUrl(req.url())}`, start: Date.now() });
|
||||
});
|
||||
page.on('requestfinished', async (req) => {
|
||||
const rec = inFlight.get(req);
|
||||
inFlight.delete(req);
|
||||
const status = (await req.response().catch(() => null))?.status() ?? '???';
|
||||
const label = rec?.label ?? `${req.method()} ${sanitizeUrl(req.url())}`;
|
||||
log(`REQ ${status} ${label} (${rec ? Date.now() - rec.start : '?'}ms)`);
|
||||
});
|
||||
page.on('requestfailed', (req) => {
|
||||
const rec = inFlight.get(req);
|
||||
inFlight.delete(req);
|
||||
const label = rec?.label ?? `${req.method()} ${sanitizeUrl(req.url())}`;
|
||||
log(`REQFAIL ${label} ${req.failure()?.errorText ?? ''} (${rec ? Date.now() - rec.start : '?'}ms)`);
|
||||
});
|
||||
page.on('console', (msg) => {
|
||||
if (msg.type() === 'error' || msg.type() === 'warning') log(`CONSOLE.${msg.type()} ${clip(msg.text())}`);
|
||||
});
|
||||
page.on('pageerror', (err) => log(`PAGEERROR ${clip(`${err.name}: ${err.message}`)}`));
|
||||
|
||||
const pending = () => {
|
||||
const now = Date.now();
|
||||
return [...inFlight.values()].map((r) => `${r.label} (${now - r.start}ms)`);
|
||||
};
|
||||
const lines = () => (dropped ? [...entries, `… ${dropped} more entries dropped`] : [...entries]);
|
||||
|
||||
return {
|
||||
lines,
|
||||
pending,
|
||||
async writeOnFailure(testInfo) {
|
||||
if (testInfo.status === testInfo.expectedStatus) return;
|
||||
const stillPending = pending();
|
||||
const body = [
|
||||
...lines(),
|
||||
'',
|
||||
`IN FLIGHT AT FAILURE (${stillPending.length}):`,
|
||||
...stillPending.map((p) => ` ${p}`),
|
||||
].join('\n');
|
||||
const file = testInfo.outputPath('page-activity.log');
|
||||
fs.writeFileSync(file, `${body}\n`);
|
||||
await testInfo.attach('page-activity', { path: file, contentType: 'text/plain' });
|
||||
console.log(`[page-activity] ${stillPending.length} request(s) in flight at failure:`);
|
||||
for (const p of stillPending) console.log(`[page-activity] ${p}`);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import * as path from 'path';
|
||||
import * as fs from 'fs';
|
||||
import { createEmailHelper, isEmailTestingAvailable, EmailHelper } from '../helpers/email-testing';
|
||||
import { clickCompleteSignInGate } from '../helpers/auth';
|
||||
import { recordPageActivity, PageActivityRecorder } from '../helpers/page-activity';
|
||||
|
||||
// Real test audio (mixed + instrumental), gitignored under e2e/fixtures.
|
||||
// Populate with scripts/e2e/fetch_test_audio.sh (CI + local).
|
||||
@@ -43,10 +44,22 @@ test.describe(`Tenant Portal Happy Path — ${TENANT_ID}`, () => {
|
||||
let emailHelper: EmailHelper | null = null;
|
||||
let createdJobId: string | null = null;
|
||||
|
||||
let activity: PageActivityRecorder | null = null;
|
||||
|
||||
test.beforeAll(async () => {
|
||||
if (isEmailTestingAvailable()) emailHelper = await createEmailHelper();
|
||||
});
|
||||
|
||||
// Traces are off on CI (public artifacts), so on failure leave a sanitized
|
||||
// request/console log showing what the page was waiting on (page-activity.ts).
|
||||
test.beforeEach(({ page }) => {
|
||||
activity = recordPageActivity(page);
|
||||
});
|
||||
|
||||
test.afterEach(async ({}, testInfo) => {
|
||||
await activity?.writeOnFailure(testInfo);
|
||||
});
|
||||
|
||||
test.afterAll(async () => {
|
||||
// E2E_KEEP_JOB=1 keeps the job + outputs for manual inspection (local runs only).
|
||||
if (process.env.E2E_KEEP_JOB === '1') {
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
// frontend/e2e/self-healing/page-activity.spec.ts — tests for e2e/helpers/page-activity.ts
|
||||
// (lives in this offline suite because it runs in CI with a real browser and no network).
|
||||
import { test, expect } from '@playwright/test';
|
||||
import * as fs from 'fs';
|
||||
import { recordPageActivity, redactText, sanitizeUrl } from '../helpers/page-activity';
|
||||
|
||||
test('sanitizeUrl keeps origin + path and drops query strings and fragments', () => {
|
||||
expect(sanitizeUrl('https://gen.example.test/auth/verify?token=secret-magic#x')).toBe('https://gen.example.test/auth/verify');
|
||||
expect(sanitizeUrl('https://storage.example.test/b/o.png?X-Goog-Signature=abc')).toBe('https://storage.example.test/b/o.png');
|
||||
expect(sanitizeUrl('data:image/png;base64,AAAA')).toBe('data:…');
|
||||
expect(sanitizeUrl('not a url?token=x')).toBe('not a url');
|
||||
});
|
||||
|
||||
test('redactText strips URL query strings, token params and bearer tokens from free text', () => {
|
||||
const out = redactText(
|
||||
'GET https://storage.example.test/b/o.mp3?X-Goog-Signature=sig123 failed; token=abc123 key: "k-999" Bearer eyJhbGc.x.y',
|
||||
);
|
||||
expect(out).toContain('https://storage.example.test/b/o.mp3');
|
||||
for (const leaked of ['sig123', 'abc123', 'k-999', 'eyJhbGc']) expect(out).not.toContain(leaked);
|
||||
});
|
||||
|
||||
test('records navigations, requests, failures, console + page errors, and in-flight requests', async ({ page }, testInfo) => {
|
||||
const ORIGIN = 'https://portal.example.test';
|
||||
await page.route(`${ORIGIN}/app`, (route) =>
|
||||
route.fulfill({
|
||||
contentType: 'text/html',
|
||||
body: `<html><body><script>
|
||||
fetch('${ORIGIN}/api/config?token=secret-magic');
|
||||
fetch('${ORIGIN}/api/broken').catch(() => {});
|
||||
fetch('${ORIGIN}/api/hangs');
|
||||
console.error('boom from console https://cdn.example.test/x.js?token=secret-console');
|
||||
setTimeout(() => { throw new Error('uncaught kaboom token=secret-pageerror'); }, 0);
|
||||
</script></body></html>`,
|
||||
}),
|
||||
);
|
||||
await page.route(`${ORIGIN}/api/config*`, (route) => route.fulfill({ status: 200, body: '{}' }));
|
||||
await page.route(`${ORIGIN}/api/broken`, (route) => route.abort('failed'));
|
||||
await page.route(`${ORIGIN}/api/hangs`, () => { /* never answered: stays in flight */ });
|
||||
|
||||
const activity = recordPageActivity(page);
|
||||
await page.goto(`${ORIGIN}/app`);
|
||||
await expect.poll(() => activity.lines().join('\n')).toContain('PAGEERROR');
|
||||
await expect.poll(() => activity.lines().join('\n')).toContain('REQ 200 GET https://portal.example.test/api/config');
|
||||
await expect.poll(() => activity.lines().join('\n')).toContain('REQFAIL');
|
||||
|
||||
const text = activity.lines().join('\n');
|
||||
expect(text).toContain('NAV https://portal.example.test/app');
|
||||
expect(text).toMatch(/REQFAIL {2}GET https:\/\/portal\.example\.test\/api\/broken/);
|
||||
expect(text).toContain('CONSOLE.error boom from console https://cdn.example.test/x.js');
|
||||
expect(text).toContain('PAGEERROR Error: uncaught kaboom token=[redacted]');
|
||||
for (const leaked of ['secret-magic', 'secret-console', 'secret-pageerror']) expect(text).not.toContain(leaked);
|
||||
expect(activity.pending().join('\n')).toContain('GET https://portal.example.test/api/hangs');
|
||||
|
||||
// Passing test: nothing written.
|
||||
await activity.writeOnFailure(testInfo);
|
||||
expect(fs.existsSync(testInfo.outputPath('page-activity.log'))).toBe(false);
|
||||
|
||||
// Failed test: log written with the in-flight section.
|
||||
const failed = Object.create(testInfo, { status: { value: 'timedOut' } });
|
||||
await activity.writeOnFailure(failed);
|
||||
const log = fs.readFileSync(testInfo.outputPath('page-activity.log'), 'utf8');
|
||||
expect(log).toContain('IN FLIGHT AT FAILURE (1):');
|
||||
expect(log).toContain('/api/hangs');
|
||||
expect(log).not.toContain('secret-magic');
|
||||
});
|
||||
Reference in new issue
Block a user