308 lines
8.5 KiB
TypeScript
308 lines
8.5 KiB
TypeScript
import { expect, test } from '@playwright/test';
|
|
import { mkdir, writeFile } from 'node:fs/promises';
|
|
import path from 'node:path';
|
|
import { dev_api_base, testing_account_id, testing_event_id } from './_helpers/env';
|
|
|
|
const testing_journal_id = 'BVYE-94-46-29';
|
|
const apiSecretKey =
|
|
process.env.PUBLIC_AE_API_SECRET_KEY ?? process.env.AE_API_SECRET_KEY ?? '';
|
|
const probeEnabled = process.env.RUN_V3_LATENCY_PROBE === '1';
|
|
const outputDir = process.env.V3_LATENCY_OUTPUT_DIR ?? 'tests/results';
|
|
|
|
type ProbeSample = {
|
|
label: string;
|
|
ms: number;
|
|
rows: number;
|
|
status: number;
|
|
ok: boolean;
|
|
error?: string;
|
|
};
|
|
|
|
type EndpointProbe = {
|
|
name: 'event_sessions' | 'journal_entries' | 'users';
|
|
label: string;
|
|
url: string;
|
|
body?: unknown;
|
|
};
|
|
|
|
function percentile(values: number[], pct: number): number {
|
|
if (values.length === 0) return 0;
|
|
const sorted = [...values].sort((a, b) => a - b);
|
|
const idx = Math.min(sorted.length - 1, Math.max(0, Math.ceil((pct / 100) * sorted.length) - 1));
|
|
return sorted[idx];
|
|
}
|
|
|
|
function summarize(samples: ProbeSample[]) {
|
|
const timings = samples.map((sample) => sample.ms);
|
|
const statuses = samples.map((sample) => sample.status);
|
|
const ok_count = samples.filter((sample) => sample.ok).length;
|
|
const error_count = samples.length - ok_count;
|
|
const row_counts = samples.map((sample) => sample.rows);
|
|
const total = timings.reduce((sum, value) => sum + value, 0);
|
|
return {
|
|
count: samples.length,
|
|
ok_count,
|
|
error_count,
|
|
error_rate: Number((error_count / Math.max(1, samples.length)).toFixed(4)),
|
|
min: Math.min(...timings),
|
|
p50: percentile(timings, 50),
|
|
p95: percentile(timings, 95),
|
|
max: Math.max(...timings),
|
|
avg: Math.round(total / Math.max(1, timings.length)),
|
|
rows_last: samples.at(-1)?.rows ?? 0,
|
|
rows_min: Math.min(...row_counts),
|
|
rows_max: Math.max(...row_counts),
|
|
statuses
|
|
};
|
|
}
|
|
|
|
async function timedJsonFetch({
|
|
label,
|
|
url,
|
|
headers,
|
|
body
|
|
}: {
|
|
label: string;
|
|
url: string;
|
|
headers: Record<string, string>;
|
|
body?: unknown;
|
|
}): Promise<ProbeSample> {
|
|
const started_ms = performance.now();
|
|
try {
|
|
const response = await fetch(url, {
|
|
method: body ? 'POST' : 'GET',
|
|
headers,
|
|
body: body ? JSON.stringify(body) : undefined
|
|
});
|
|
const elapsed_ms = Math.round(performance.now() - started_ms);
|
|
|
|
const payload = await response.json().catch(() => null);
|
|
const rows = Array.isArray(payload?.data)
|
|
? payload.data.length
|
|
: Array.isArray(payload)
|
|
? payload.length
|
|
: 0;
|
|
|
|
return {
|
|
label,
|
|
ms: elapsed_ms,
|
|
rows,
|
|
status: response.status,
|
|
ok: response.ok
|
|
};
|
|
} catch (error) {
|
|
const elapsed_ms = Math.round(performance.now() - started_ms);
|
|
return {
|
|
label,
|
|
ms: elapsed_ms,
|
|
rows: 0,
|
|
status: 0,
|
|
ok: false,
|
|
error: error instanceof Error ? error.message : String(error)
|
|
};
|
|
}
|
|
}
|
|
|
|
function reportMarkdown({
|
|
run_id,
|
|
started_at,
|
|
base_url,
|
|
rounds,
|
|
pause_ms,
|
|
threshold_max_error_rate,
|
|
threshold_p95_ms,
|
|
require_non_empty_rows,
|
|
report,
|
|
anomalies
|
|
}: any): string {
|
|
const lines: string[] = [];
|
|
lines.push('# V3 API Performance Probe');
|
|
lines.push('');
|
|
lines.push(`- run_id: ${run_id}`);
|
|
lines.push(`- started_at: ${started_at}`);
|
|
lines.push(`- base_url: ${base_url}`);
|
|
lines.push(`- rounds_per_endpoint: ${rounds}`);
|
|
lines.push(`- pause_ms: ${pause_ms}`);
|
|
lines.push(`- threshold_max_error_rate: ${threshold_max_error_rate}`);
|
|
lines.push(`- threshold_p95_ms: ${threshold_p95_ms ?? 'disabled'}`);
|
|
lines.push(`- require_non_empty_rows: ${require_non_empty_rows}`);
|
|
lines.push('');
|
|
lines.push('| Endpoint | count | errors | error_rate | p50 | p95 | max | rows_min | rows_max |');
|
|
lines.push('| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |');
|
|
|
|
for (const [name, stats] of Object.entries(report) as any) {
|
|
lines.push(
|
|
`| ${name} | ${stats.count} | ${stats.error_count} | ${stats.error_rate} | ${stats.p50} | ${stats.p95} | ${stats.max} | ${stats.rows_min} | ${stats.rows_max} |`
|
|
);
|
|
}
|
|
|
|
lines.push('');
|
|
if (anomalies.length > 0) {
|
|
lines.push('## Anomalies');
|
|
for (const item of anomalies) lines.push(`- ${item}`);
|
|
} else {
|
|
lines.push('## Anomalies');
|
|
lines.push('- none');
|
|
}
|
|
|
|
return lines.join('\n');
|
|
}
|
|
|
|
test.describe('V3 API latency probe', () => {
|
|
test.skip(
|
|
!probeEnabled || !apiSecretKey,
|
|
'Set RUN_V3_LATENCY_PROBE=1 and PUBLIC_AE_API_SECRET_KEY to run the live probe.'
|
|
);
|
|
test.setTimeout(120000);
|
|
|
|
test('quick rounds on common list endpoints', async () => {
|
|
const rounds = Number(process.env.V3_LATENCY_ROUNDS ?? 6);
|
|
const delay_ms = Number(process.env.V3_LATENCY_PAUSE_MS ?? 150);
|
|
const threshold_max_error_rate = Number(process.env.V3_LATENCY_MAX_ERROR_RATE ?? 0);
|
|
const threshold_p95_ms = process.env.V3_LATENCY_MAX_P95_MS
|
|
? Number(process.env.V3_LATENCY_MAX_P95_MS)
|
|
: null;
|
|
const require_non_empty_rows = process.env.V3_LATENCY_REQUIRE_ROWS === '1';
|
|
const started_at = new Date().toISOString();
|
|
const run_id = started_at.replace(/[:.]/g, '-');
|
|
const headers = {
|
|
'x-aether-api-key': apiSecretKey,
|
|
'x-account-id': testing_account_id,
|
|
'x-ae-ignore-extra-fields': 'true',
|
|
'Content-Type': 'application/json'
|
|
};
|
|
|
|
const event_session_url = new URL('/v3/crud/event_session/search', dev_api_base).toString();
|
|
const journal_entry_url = new URL(`/v3/crud/journal/${testing_journal_id}/journal_entry/`, dev_api_base).toString();
|
|
const user_list_url = new URL('/v3/crud/user/', dev_api_base).toString();
|
|
|
|
const probes: EndpointProbe[] = [
|
|
{
|
|
name: 'event_sessions',
|
|
label: 'event_session',
|
|
url: event_session_url,
|
|
body: {
|
|
and: [{ field: 'event_id', op: 'eq', value: testing_event_id }]
|
|
}
|
|
},
|
|
{
|
|
name: 'journal_entries',
|
|
label: 'journal_entry',
|
|
url: journal_entry_url
|
|
},
|
|
{
|
|
name: 'users',
|
|
label: 'user',
|
|
url: `${user_list_url}?${new URLSearchParams({
|
|
for_obj_type: 'account',
|
|
for_obj_id: testing_account_id,
|
|
enabled: 'all',
|
|
hidden: 'not_hidden',
|
|
view: 'default',
|
|
limit: '99',
|
|
offset: '0',
|
|
order_by_li: JSON.stringify({ username: 'ASC' })
|
|
}).toString()}`
|
|
}
|
|
];
|
|
|
|
const samples_by_endpoint: Record<string, ProbeSample[]> = {
|
|
event_sessions: [],
|
|
journal_entries: [],
|
|
users: []
|
|
};
|
|
|
|
for (let round = 1; round <= rounds; round++) {
|
|
for (const probe of probes) {
|
|
samples_by_endpoint[probe.name].push(
|
|
await timedJsonFetch({
|
|
label: `${probe.label} round ${round}`,
|
|
url: probe.url,
|
|
headers,
|
|
body: probe.body
|
|
})
|
|
);
|
|
|
|
await new Promise((resolve) => setTimeout(resolve, delay_ms));
|
|
}
|
|
}
|
|
|
|
const report = {
|
|
event_sessions: summarize(samples_by_endpoint.event_sessions),
|
|
journal_entries: summarize(samples_by_endpoint.journal_entries),
|
|
users: summarize(samples_by_endpoint.users)
|
|
};
|
|
|
|
const anomalies: string[] = [];
|
|
for (const [name, stats] of Object.entries(report) as any) {
|
|
if (stats.error_rate > threshold_max_error_rate) {
|
|
anomalies.push(
|
|
`${name}: error_rate ${stats.error_rate} > threshold ${threshold_max_error_rate}`
|
|
);
|
|
}
|
|
if (threshold_p95_ms !== null && stats.p95 > threshold_p95_ms) {
|
|
anomalies.push(
|
|
`${name}: p95 ${stats.p95}ms > threshold ${threshold_p95_ms}ms`
|
|
);
|
|
}
|
|
if (require_non_empty_rows && stats.rows_max === 0) {
|
|
anomalies.push(`${name}: all rounds returned 0 rows`);
|
|
}
|
|
if (stats.rows_max > 0 && stats.rows_min === 0) {
|
|
anomalies.push(
|
|
`${name}: row count flapped between empty and non-empty (rows_min=0 rows_max=${stats.rows_max})`
|
|
);
|
|
}
|
|
if (stats.p95 > stats.p50 * 3 && stats.p95 > 1000) {
|
|
anomalies.push(
|
|
`${name}: jitter spike (p95=${stats.p95}ms vs p50=${stats.p50}ms)`
|
|
);
|
|
}
|
|
}
|
|
|
|
const report_payload = {
|
|
run_id,
|
|
started_at,
|
|
base_url: dev_api_base,
|
|
rounds,
|
|
pause_ms: delay_ms,
|
|
threshold_max_error_rate,
|
|
threshold_p95_ms,
|
|
require_non_empty_rows,
|
|
report,
|
|
samples: samples_by_endpoint,
|
|
anomalies
|
|
};
|
|
|
|
await mkdir(outputDir, { recursive: true });
|
|
const json_path = path.join(outputDir, `v3_latency_probe_${run_id}.json`);
|
|
const md_path = path.join(outputDir, `v3_latency_probe_${run_id}.md`);
|
|
|
|
await writeFile(json_path, `${JSON.stringify(report_payload, null, 2)}\n`, 'utf8');
|
|
await writeFile(
|
|
md_path,
|
|
reportMarkdown({
|
|
run_id,
|
|
started_at,
|
|
base_url: dev_api_base,
|
|
rounds,
|
|
pause_ms: delay_ms,
|
|
threshold_max_error_rate,
|
|
threshold_p95_ms,
|
|
require_non_empty_rows,
|
|
report,
|
|
anomalies
|
|
}),
|
|
'utf8'
|
|
);
|
|
|
|
console.log('V3 latency probe summary:');
|
|
console.table(report);
|
|
console.log('V3 latency probe report files:', {
|
|
json_path,
|
|
md_path
|
|
});
|
|
|
|
expect(anomalies, `Latency probe anomalies:\n- ${anomalies.join('\n- ')}`).toEqual([]);
|
|
});
|
|
}); |