From 55d3d49595d1dd520b3e26a0e8e66bc322dbb63b Mon Sep 17 00:00:00 2001 From: Scott Idem Date: Thu, 21 May 2026 17:48:00 -0400 Subject: [PATCH] test: add v3 latency probe and modernize api coverage --- tests/README.md | 15 ++ tests/v3_api_latency_probe.test.ts | 308 +++++++++++++++++++++++++++ tests/v3_api_nested_crud.test.ts | 47 ++-- tests/v3_api_security.modern.test.ts | 7 +- 4 files changed, 341 insertions(+), 36 deletions(-) create mode 100644 tests/v3_api_latency_probe.test.ts diff --git a/tests/README.md b/tests/README.md index 6554f4c9..29fc8c70 100644 --- a/tests/README.md +++ b/tests/README.md @@ -74,6 +74,21 @@ git add tests/ git commit -m "test: add " ``` +Latency probing +- Use the gated probe in `tests/v3_api_latency_probe.test.ts` for quick live rounds against V3 list endpoints. +- Run it only when you have the live API key available: + +```bash +RUN_V3_LATENCY_PROBE=1 PUBLIC_AE_API_SECRET_KEY=... npx playwright test tests/v3_api_latency_probe.test.ts -c playwright.config.ts +``` +- Tune the rounds with `V3_LATENCY_ROUNDS` and the pause between calls with `V3_LATENCY_PAUSE_MS`. +- Reports are written to `tests/results/` as JSON and Markdown per run. +- Optional bug-finding thresholds: + - `V3_LATENCY_MAX_ERROR_RATE` (default `0`) — fail if an endpoint exceeds this error rate + - `V3_LATENCY_MAX_P95_MS` (optional) — fail if endpoint p95 exceeds the threshold + - `V3_LATENCY_REQUIRE_ROWS=1` (optional) — fail if all rounds return zero rows + - `V3_LATENCY_OUTPUT_DIR` (optional) — override report directory (default `tests/results`) + Help - If a test fails due to external network calls or platform-specific behavior, try mocking the relevant endpoints and move the test to `tests/disabled` if it cannot be made deterministic. diff --git a/tests/v3_api_latency_probe.test.ts b/tests/v3_api_latency_probe.test.ts new file mode 100644 index 00000000..e20f8f46 --- /dev/null +++ b/tests/v3_api_latency_probe.test.ts @@ -0,0 +1,308 @@ +import { expect, test } from '@playwright/test'; +import { mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { dev_api_base, testing_account_id, testing_event_id } from './_helpers/env'; + +const testing_journal_id = 'BVYE-94-46-29'; +const apiSecretKey = + process.env.PUBLIC_AE_API_SECRET_KEY ?? process.env.AE_API_SECRET_KEY ?? ''; +const probeEnabled = process.env.RUN_V3_LATENCY_PROBE === '1'; +const outputDir = process.env.V3_LATENCY_OUTPUT_DIR ?? 'tests/results'; + +type ProbeSample = { + label: string; + ms: number; + rows: number; + status: number; + ok: boolean; + error?: string; +}; + +type EndpointProbe = { + name: 'event_sessions' | 'journal_entries' | 'users'; + label: string; + url: string; + body?: unknown; +}; + +function percentile(values: number[], pct: number): number { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((pct / 100) * sorted.length) - 1)); + return sorted[idx]; +} + +function summarize(samples: ProbeSample[]) { + const timings = samples.map((sample) => sample.ms); + const statuses = samples.map((sample) => sample.status); + const ok_count = samples.filter((sample) => sample.ok).length; + const error_count = samples.length - ok_count; + const row_counts = samples.map((sample) => sample.rows); + const total = timings.reduce((sum, value) => sum + value, 0); + return { + count: samples.length, + ok_count, + error_count, + error_rate: Number((error_count / Math.max(1, samples.length)).toFixed(4)), + min: Math.min(...timings), + p50: percentile(timings, 50), + p95: percentile(timings, 95), + max: Math.max(...timings), + avg: Math.round(total / Math.max(1, timings.length)), + rows_last: samples.at(-1)?.rows ?? 0, + rows_min: Math.min(...row_counts), + rows_max: Math.max(...row_counts), + statuses + }; +} + +async function timedJsonFetch({ + label, + url, + headers, + body +}: { + label: string; + url: string; + headers: Record; + body?: unknown; +}): Promise { + const started_ms = performance.now(); + try { + const response = await fetch(url, { + method: body ? 'POST' : 'GET', + headers, + body: body ? JSON.stringify(body) : undefined + }); + const elapsed_ms = Math.round(performance.now() - started_ms); + + const payload = await response.json().catch(() => null); + const rows = Array.isArray(payload?.data) + ? payload.data.length + : Array.isArray(payload) + ? payload.length + : 0; + + return { + label, + ms: elapsed_ms, + rows, + status: response.status, + ok: response.ok + }; + } catch (error) { + const elapsed_ms = Math.round(performance.now() - started_ms); + return { + label, + ms: elapsed_ms, + rows: 0, + status: 0, + ok: false, + error: error instanceof Error ? error.message : String(error) + }; + } +} + +function reportMarkdown({ + run_id, + started_at, + base_url, + rounds, + pause_ms, + threshold_max_error_rate, + threshold_p95_ms, + require_non_empty_rows, + report, + anomalies +}: any): string { + const lines: string[] = []; + lines.push('# V3 API Performance Probe'); + lines.push(''); + lines.push(`- run_id: ${run_id}`); + lines.push(`- started_at: ${started_at}`); + lines.push(`- base_url: ${base_url}`); + lines.push(`- rounds_per_endpoint: ${rounds}`); + lines.push(`- pause_ms: ${pause_ms}`); + lines.push(`- threshold_max_error_rate: ${threshold_max_error_rate}`); + lines.push(`- threshold_p95_ms: ${threshold_p95_ms ?? 'disabled'}`); + lines.push(`- require_non_empty_rows: ${require_non_empty_rows}`); + lines.push(''); + lines.push('| Endpoint | count | errors | error_rate | p50 | p95 | max | rows_min | rows_max |'); + lines.push('| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |'); + + for (const [name, stats] of Object.entries(report) as any) { + lines.push( + `| ${name} | ${stats.count} | ${stats.error_count} | ${stats.error_rate} | ${stats.p50} | ${stats.p95} | ${stats.max} | ${stats.rows_min} | ${stats.rows_max} |` + ); + } + + lines.push(''); + if (anomalies.length > 0) { + lines.push('## Anomalies'); + for (const item of anomalies) lines.push(`- ${item}`); + } else { + lines.push('## Anomalies'); + lines.push('- none'); + } + + return lines.join('\n'); +} + +test.describe('V3 API latency probe', () => { + test.skip( + !probeEnabled || !apiSecretKey, + 'Set RUN_V3_LATENCY_PROBE=1 and PUBLIC_AE_API_SECRET_KEY to run the live probe.' + ); + test.setTimeout(120000); + + test('quick rounds on common list endpoints', async () => { + const rounds = Number(process.env.V3_LATENCY_ROUNDS ?? 6); + const delay_ms = Number(process.env.V3_LATENCY_PAUSE_MS ?? 150); + const threshold_max_error_rate = Number(process.env.V3_LATENCY_MAX_ERROR_RATE ?? 0); + const threshold_p95_ms = process.env.V3_LATENCY_MAX_P95_MS + ? Number(process.env.V3_LATENCY_MAX_P95_MS) + : null; + const require_non_empty_rows = process.env.V3_LATENCY_REQUIRE_ROWS === '1'; + const started_at = new Date().toISOString(); + const run_id = started_at.replace(/[:.]/g, '-'); + const headers = { + 'x-aether-api-key': apiSecretKey, + 'x-account-id': testing_account_id, + 'x-ae-ignore-extra-fields': 'true', + 'Content-Type': 'application/json' + }; + + const event_session_url = new URL('/v3/crud/event_session/search', dev_api_base).toString(); + const journal_entry_url = new URL(`/v3/crud/journal/${testing_journal_id}/journal_entry/`, dev_api_base).toString(); + const user_list_url = new URL('/v3/crud/user/', dev_api_base).toString(); + + const probes: EndpointProbe[] = [ + { + name: 'event_sessions', + label: 'event_session', + url: event_session_url, + body: { + and: [{ field: 'event_id', op: 'eq', value: testing_event_id }] + } + }, + { + name: 'journal_entries', + label: 'journal_entry', + url: journal_entry_url + }, + { + name: 'users', + label: 'user', + url: `${user_list_url}?${new URLSearchParams({ + for_obj_type: 'account', + for_obj_id: testing_account_id, + enabled: 'all', + hidden: 'not_hidden', + view: 'default', + limit: '99', + offset: '0', + order_by_li: JSON.stringify({ username: 'ASC' }) + }).toString()}` + } + ]; + + const samples_by_endpoint: Record = { + event_sessions: [], + journal_entries: [], + users: [] + }; + + for (let round = 1; round <= rounds; round++) { + for (const probe of probes) { + samples_by_endpoint[probe.name].push( + await timedJsonFetch({ + label: `${probe.label} round ${round}`, + url: probe.url, + headers, + body: probe.body + }) + ); + + await new Promise((resolve) => setTimeout(resolve, delay_ms)); + } + } + + const report = { + event_sessions: summarize(samples_by_endpoint.event_sessions), + journal_entries: summarize(samples_by_endpoint.journal_entries), + users: summarize(samples_by_endpoint.users) + }; + + const anomalies: string[] = []; + for (const [name, stats] of Object.entries(report) as any) { + if (stats.error_rate > threshold_max_error_rate) { + anomalies.push( + `${name}: error_rate ${stats.error_rate} > threshold ${threshold_max_error_rate}` + ); + } + if (threshold_p95_ms !== null && stats.p95 > threshold_p95_ms) { + anomalies.push( + `${name}: p95 ${stats.p95}ms > threshold ${threshold_p95_ms}ms` + ); + } + if (require_non_empty_rows && stats.rows_max === 0) { + anomalies.push(`${name}: all rounds returned 0 rows`); + } + if (stats.rows_max > 0 && stats.rows_min === 0) { + anomalies.push( + `${name}: row count flapped between empty and non-empty (rows_min=0 rows_max=${stats.rows_max})` + ); + } + if (stats.p95 > stats.p50 * 3 && stats.p95 > 1000) { + anomalies.push( + `${name}: jitter spike (p95=${stats.p95}ms vs p50=${stats.p50}ms)` + ); + } + } + + const report_payload = { + run_id, + started_at, + base_url: dev_api_base, + rounds, + pause_ms: delay_ms, + threshold_max_error_rate, + threshold_p95_ms, + require_non_empty_rows, + report, + samples: samples_by_endpoint, + anomalies + }; + + await mkdir(outputDir, { recursive: true }); + const json_path = path.join(outputDir, `v3_latency_probe_${run_id}.json`); + const md_path = path.join(outputDir, `v3_latency_probe_${run_id}.md`); + + await writeFile(json_path, `${JSON.stringify(report_payload, null, 2)}\n`, 'utf8'); + await writeFile( + md_path, + reportMarkdown({ + run_id, + started_at, + base_url: dev_api_base, + rounds, + pause_ms: delay_ms, + threshold_max_error_rate, + threshold_p95_ms, + require_non_empty_rows, + report, + anomalies + }), + 'utf8' + ); + + console.log('V3 latency probe summary:'); + console.table(report); + console.log('V3 latency probe report files:', { + json_path, + md_path + }); + + expect(anomalies, `Latency probe anomalies:\n- ${anomalies.join('\n- ')}`).toEqual([]); + }); +}); \ No newline at end of file diff --git a/tests/v3_api_nested_crud.test.ts b/tests/v3_api_nested_crud.test.ts index 104a047a..ffbb7a60 100644 --- a/tests/v3_api_nested_crud.test.ts +++ b/tests/v3_api_nested_crud.test.ts @@ -101,46 +101,25 @@ test.describe('V3 API Nested CRUD Integrity', () => { }); test('should send a nested request when creating an Event Location', async ({ page }) => { - // We'll perform the UI action and assert the resulting UI change (and the route handler - // separately logs the POST). Relying on DOM update is less flaky than waiting - // directly for the network request in this environment. - - // The page is now loaded. The test will automatically fail because - // the UI is not yet interactive enough to trigger the POST request. - // The console output will show us which GET requests we need to mock. - + // Validate the real app flow: click the UI button and assert the outgoing + // nested POST request shape and endpoint. + const requestPromise = page.waitForRequest( + (request) => + request.method() === 'POST' && + request.url().includes(`/v3/crud/event/${testing_event_id}/event_location`) + ); // Ensure the Add Location button is present const addBtn = page.getByRole('button', { name: 'Add Location' }); await expect(addBtn).toBeVisible(); + await addBtn.click(); - // Instead of relying on the complex client-side helper to call the nested create, - // POST directly from the browser context to the nested endpoint so the page.route - // handler is exercised and we can assert nested endpoint behavior. - const resp = await page.evaluate(async (eventId) => { - const r = await fetch(`/v3/crud/event/${eventId}/event_location/`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ name: 'TEMP Location Name', event_id: eventId }) - }); - try { return { status: r.status, json: await r.json() }; } catch(e) { return { status: r.status, json: null }; } - }, testing_event_id as any); + const request = await requestPromise; + const postData = JSON.parse(request.postData() ?? '{}'); - expect(resp.status === 200 || resp.status === 201).toBeTruthy(); - expect(resp.json).toBeDefined(); - if (resp.json && resp.json.data) expect(resp.json.data.name).toBe('TEMP Location Name'); + expect(request.url()).toContain(`/v3/crud/event/${testing_event_id}/event_location`); + expect(postData.name).toBe('TEMP Location Name'); + expect(postData.event_id).toBe(testing_event_id); - - // Wait for the request to be captured - // const request = await requestPromise; - // const postData = request.postDataJSON(); - - // Assert that the request was sent to the correct nested URL - // expect(request.url()).toContain(`/v3/crud/event/${testing_event_id}/event_location`); - - // Assert that the payload contains the correct fields and *does not* contain the parent ID - // expect(postData.fields).toBeDefined(); - // expect(postData.fields.name).toBe('Test Location'); - // expect(postData.fields.event_id).toBeUndefined(); }); }); diff --git a/tests/v3_api_security.modern.test.ts b/tests/v3_api_security.modern.test.ts index 8d7c2966..09f5462f 100644 --- a/tests/v3_api_security.modern.test.ts +++ b/tests/v3_api_security.modern.test.ts @@ -37,7 +37,7 @@ test.describe('V3 API Header Integrity (modernized)', () => { }); }); - test('Verify lookup requests include the unauthenticated bypass header', async ({ page }) => { + test('Verify lookup requests use account-scoped headers (no bypass)', async ({ page }) => { await page.addInitScript((defaults) => { const testData = { ...defaults, account_id: 'test-account-id', manager_access: true }; window.localStorage.setItem('ae_loc', JSON.stringify(testData)); @@ -50,7 +50,10 @@ test.describe('V3 API Header Integrity (modernized)', () => { const request = await requestPromise; const headers = request.headers(); - expect(headers['x-no-account-id']).toBe('Nothing to See Here'); + // Current lookup policy is account-scoped for these routes. + // The bypass header should not be sent here. + expect(headers['x-no-account-id']).toBeUndefined(); + expect(headers['x-account-id']).toBe('test-account-id'); expect(headers['x-aether-api-key']).toBeDefined(); });