import { expect, test } from '@playwright/test'; import { mkdir, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { dev_api_base, testing_account_id, testing_event_id } from './_helpers/env'; const testing_journal_id = 'BVYE-94-46-29'; const apiSecretKey = process.env.PUBLIC_AE_API_SECRET_KEY ?? process.env.AE_API_SECRET_KEY ?? ''; const probeEnabled = process.env.RUN_V3_LATENCY_PROBE === '1'; const outputDir = process.env.V3_LATENCY_OUTPUT_DIR ?? 'tests/results'; type ProbeSample = { label: string; ms: number; rows: number; status: number; ok: boolean; error?: string; }; type EndpointProbe = { name: 'event_sessions' | 'journal_entries' | 'users'; label: string; url: string; body?: unknown; }; function percentile(values: number[], pct: number): number { if (values.length === 0) return 0; const sorted = [...values].sort((a, b) => a - b); const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((pct / 100) * sorted.length) - 1)); return sorted[idx]; } function summarize(samples: ProbeSample[]) { const timings = samples.map((sample) => sample.ms); const statuses = samples.map((sample) => sample.status); const ok_count = samples.filter((sample) => sample.ok).length; const error_count = samples.length - ok_count; const row_counts = samples.map((sample) => sample.rows); const total = timings.reduce((sum, value) => sum + value, 0); return { count: samples.length, ok_count, error_count, error_rate: Number((error_count / Math.max(1, samples.length)).toFixed(4)), min: Math.min(...timings), p50: percentile(timings, 50), p95: percentile(timings, 95), max: Math.max(...timings), avg: Math.round(total / Math.max(1, timings.length)), rows_last: samples.at(-1)?.rows ?? 0, rows_min: Math.min(...row_counts), rows_max: Math.max(...row_counts), statuses }; } async function timedJsonFetch({ label, url, headers, body }: { label: string; url: string; headers: Record; body?: unknown; }): Promise { const started_ms = performance.now(); try { const response = await fetch(url, { method: body ? 'POST' : 'GET', headers, body: body ? JSON.stringify(body) : undefined }); const elapsed_ms = Math.round(performance.now() - started_ms); const payload = await response.json().catch(() => null); const rows = Array.isArray(payload?.data) ? payload.data.length : Array.isArray(payload) ? payload.length : 0; return { label, ms: elapsed_ms, rows, status: response.status, ok: response.ok }; } catch (error) { const elapsed_ms = Math.round(performance.now() - started_ms); return { label, ms: elapsed_ms, rows: 0, status: 0, ok: false, error: error instanceof Error ? error.message : String(error) }; } } function reportMarkdown({ run_id, started_at, base_url, rounds, pause_ms, threshold_max_error_rate, threshold_p95_ms, require_non_empty_rows, report, anomalies }: any): string { const lines: string[] = []; lines.push('# V3 API Performance Probe'); lines.push(''); lines.push(`- run_id: ${run_id}`); lines.push(`- started_at: ${started_at}`); lines.push(`- base_url: ${base_url}`); lines.push(`- rounds_per_endpoint: ${rounds}`); lines.push(`- pause_ms: ${pause_ms}`); lines.push(`- threshold_max_error_rate: ${threshold_max_error_rate}`); lines.push(`- threshold_p95_ms: ${threshold_p95_ms ?? 'disabled'}`); lines.push(`- require_non_empty_rows: ${require_non_empty_rows}`); lines.push(''); lines.push('| Endpoint | count | errors | error_rate | p50 | p95 | max | rows_min | rows_max |'); lines.push('| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |'); for (const [name, stats] of Object.entries(report) as any) { lines.push( `| ${name} | ${stats.count} | ${stats.error_count} | ${stats.error_rate} | ${stats.p50} | ${stats.p95} | ${stats.max} | ${stats.rows_min} | ${stats.rows_max} |` ); } lines.push(''); if (anomalies.length > 0) { lines.push('## Anomalies'); for (const item of anomalies) lines.push(`- ${item}`); } else { lines.push('## Anomalies'); lines.push('- none'); } return lines.join('\n'); } test.describe('V3 API latency probe', () => { test.skip( !probeEnabled || !apiSecretKey, 'Set RUN_V3_LATENCY_PROBE=1 and PUBLIC_AE_API_SECRET_KEY to run the live probe.' ); test.setTimeout(120000); test('quick rounds on common list endpoints', async () => { const rounds = Number(process.env.V3_LATENCY_ROUNDS ?? 6); const delay_ms = Number(process.env.V3_LATENCY_PAUSE_MS ?? 150); const threshold_max_error_rate = Number(process.env.V3_LATENCY_MAX_ERROR_RATE ?? 0); const threshold_p95_ms = process.env.V3_LATENCY_MAX_P95_MS ? Number(process.env.V3_LATENCY_MAX_P95_MS) : null; const require_non_empty_rows = process.env.V3_LATENCY_REQUIRE_ROWS === '1'; const started_at = new Date().toISOString(); const run_id = started_at.replace(/[:.]/g, '-'); const headers = { 'x-aether-api-key': apiSecretKey, 'x-account-id': testing_account_id, 'x-ae-ignore-extra-fields': 'true', 'Content-Type': 'application/json' }; const event_session_url = new URL('/v3/crud/event_session/search', dev_api_base).toString(); const journal_entry_url = new URL(`/v3/crud/journal/${testing_journal_id}/journal_entry/`, dev_api_base).toString(); const user_list_url = new URL('/v3/crud/user/', dev_api_base).toString(); const probes: EndpointProbe[] = [ { name: 'event_sessions', label: 'event_session', url: event_session_url, body: { and: [{ field: 'event_id', op: 'eq', value: testing_event_id }] } }, { name: 'journal_entries', label: 'journal_entry', url: journal_entry_url }, { name: 'users', label: 'user', url: `${user_list_url}?${new URLSearchParams({ for_obj_type: 'account', for_obj_id: testing_account_id, enabled: 'all', hidden: 'not_hidden', view: 'default', limit: '99', offset: '0', order_by_li: JSON.stringify({ username: 'ASC' }) }).toString()}` } ]; const samples_by_endpoint: Record = { event_sessions: [], journal_entries: [], users: [] }; for (let round = 1; round <= rounds; round++) { for (const probe of probes) { samples_by_endpoint[probe.name].push( await timedJsonFetch({ label: `${probe.label} round ${round}`, url: probe.url, headers, body: probe.body }) ); await new Promise((resolve) => setTimeout(resolve, delay_ms)); } } const report = { event_sessions: summarize(samples_by_endpoint.event_sessions), journal_entries: summarize(samples_by_endpoint.journal_entries), users: summarize(samples_by_endpoint.users) }; const anomalies: string[] = []; for (const [name, stats] of Object.entries(report) as any) { if (stats.error_rate > threshold_max_error_rate) { anomalies.push( `${name}: error_rate ${stats.error_rate} > threshold ${threshold_max_error_rate}` ); } if (threshold_p95_ms !== null && stats.p95 > threshold_p95_ms) { anomalies.push( `${name}: p95 ${stats.p95}ms > threshold ${threshold_p95_ms}ms` ); } if (require_non_empty_rows && stats.rows_max === 0) { anomalies.push(`${name}: all rounds returned 0 rows`); } if (stats.rows_max > 0 && stats.rows_min === 0) { anomalies.push( `${name}: row count flapped between empty and non-empty (rows_min=0 rows_max=${stats.rows_max})` ); } if (stats.p95 > stats.p50 * 3 && stats.p95 > 1000) { anomalies.push( `${name}: jitter spike (p95=${stats.p95}ms vs p50=${stats.p50}ms)` ); } } const report_payload = { run_id, started_at, base_url: dev_api_base, rounds, pause_ms: delay_ms, threshold_max_error_rate, threshold_p95_ms, require_non_empty_rows, report, samples: samples_by_endpoint, anomalies }; await mkdir(outputDir, { recursive: true }); const json_path = path.join(outputDir, `v3_latency_probe_${run_id}.json`); const md_path = path.join(outputDir, `v3_latency_probe_${run_id}.md`); await writeFile(json_path, `${JSON.stringify(report_payload, null, 2)}\n`, 'utf8'); await writeFile( md_path, reportMarkdown({ run_id, started_at, base_url: dev_api_base, rounds, pause_ms: delay_ms, threshold_max_error_rate, threshold_p95_ms, require_non_empty_rows, report, anomalies }), 'utf8' ); console.log('V3 latency probe summary:'); console.table(report); console.log('V3 latency probe report files:', { json_path, md_path }); expect(anomalies, `Latency probe anomalies:\n- ${anomalies.join('\n- ')}`).toEqual([]); }); });