diff --git a/.github/workflows/api-benchmark-smoke.yml b/.github/workflows/api-benchmark-smoke.yml new file mode 100644 index 000000000..a4da61ed6 --- /dev/null +++ b/.github/workflows/api-benchmark-smoke.yml @@ -0,0 +1,23 @@ +name: API Benchmark Smoke + +on: + pull_request: + paths: + - "apps/api/**" + - "benchmarks/**" + - "package.json" + - "package-lock.json" + - ".github/workflows/api-benchmark-smoke.yml" + +jobs: + smoke: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 24 + cache: npm + - run: npm ci + - run: npm run benchmark:smoke diff --git a/benchmarks/.env.benchmark.example b/benchmarks/.env.benchmark.example new file mode 100644 index 000000000..c26fb5f21 --- /dev/null +++ b/benchmarks/.env.benchmark.example @@ -0,0 +1,7 @@ +BENCHMARK_TARGET_URL= +BENCHMARK_AUTH_TOKEN= +BENCHMARK_ITERATIONS=20 +BENCHMARK_CONCURRENCY=4 +BENCHMARK_P99_THRESHOLD_MS=500 +BENCHMARK_ERROR_RATE_THRESHOLD=0 +JWT_SECRET=development-secret diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 000000000..a7d13d62b --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,25 @@ +# API Benchmarks + +Run a reproducible benchmark against the local Express API or an external target. + +```bash +npm run benchmark +npm run benchmark:smoke +``` + +Configuration can be supplied with environment variables. Copy `.env.benchmark.example` +when running outside the default local setup. + +- `BENCHMARK_TARGET_URL`: target host. When omitted, the runner starts the local API. +- `BENCHMARK_AUTH_TOKEN`: bearer token for protected routes. When omitted, a local admin token is generated. +- `BENCHMARK_ITERATIONS`: requests per endpoint for the full suite. +- `BENCHMARK_CONCURRENCY`: concurrent workers per endpoint. +- `BENCHMARK_P99_THRESHOLD_MS`: default p99 latency ceiling. +- `BENCHMARK_ERROR_RATE_THRESHOLD`: default allowed error rate percentage. + +Results are written to: + +- `benchmarks/results/latest.json` +- `benchmarks/results/latest.md` + +The smoke mode lowers request volume while still covering every configured endpoint. diff --git a/benchmarks/results/latest.json b/benchmarks/results/latest.json new file mode 100644 index 000000000..833cb319c --- /dev/null +++ b/benchmarks/results/latest.json @@ -0,0 +1,706 @@ +{ + "generatedAt": "2026-05-27T08:23:05.133Z", + "mode": "smoke", + "target": "local Express app", + "options": { + "iterations": 3, + "concurrency": 1 + }, + "summaries": [ + { + "name": "health", + "method": "GET", + "path": "/health", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 174.67, + "peakRps": 3, + "latencyMs": { + "p50": 1.35, + "p95": 15.13, + "p99": 15.13 + }, + "ttfbMs": { + "p50": 1.28, + "p95": 14.67, + "p99": 14.67 + }, + "statuses": { + "200": 3 + } + }, + { + "name": "auth register", + "method": "POST", + "path": "/api/auth/register", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 425.88, + "peakRps": 3, + "latencyMs": { + "p50": 0.84, + "p95": 5.54, + "p99": 5.54 + }, + "ttfbMs": { + "p50": 0.79, + "p95": 5.48, + "p99": 5.48 + }, + "statuses": { + "201": 3 + } + }, + { + "name": "auth login", + "method": "POST", + "path": "/api/auth/login", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 1850.57, + "peakRps": 3, + "latencyMs": { + "p50": 0.51, + "p95": 0.58, + "p99": 0.58 + }, + "ttfbMs": { + "p50": 0.47, + "p95": 0.54, + "p99": 0.54 + }, + "statuses": { + "200": 3 + } + }, + { + "name": "auth oauth callback", + "method": "GET", + "path": "/api/auth/oauth/github/callback", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 2566.93, + "peakRps": 3, + "latencyMs": { + "p50": 0.34, + "p95": 0.53, + "p99": 0.53 + }, + "ttfbMs": { + "p50": 0.28, + "p95": 0.5, + "p99": 0.5 + }, + "statuses": { + "200": 3 + } + }, + { + "name": "auth refresh", + "method": "POST", + "path": "/api/auth/refresh", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 2546.33, + "peakRps": 3, + "latencyMs": { + "p50": 0.4, + "p95": 0.4, + "p99": 0.4 + }, + "ttfbMs": { + "p50": 0.37, + "p95": 0.37, + "p99": 0.37 + }, + "statuses": { + "200": 3 + } + }, + { + "name": "users list", + "method": "GET", + "path": "/api/users", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 3000, + "peakRps": 3, + "latencyMs": { + "p50": 0.3, + "p95": 0.31, + "p99": 0.31 + }, + "ttfbMs": { + "p50": 0.26, + "p95": 0.29, + "p99": 0.29 + }, + "statuses": { + "200": 3 + } + }, + { + "name": "users create", + "method": "POST", + "path": "/api/users", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 2578.89, + "peakRps": 3, + "latencyMs": { + "p50": 0.37, + "p95": 0.45, + "p99": 0.45 + }, + "ttfbMs": { + "p50": 0.34, + "p95": 0.42, + "p99": 0.42 + }, + "statuses": { + "201": 3 + } + }, + { + "name": "jobs list", + "method": "GET", + "path": "/api/jobs", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 3000, + "peakRps": 3, + "latencyMs": { + "p50": 0.27, + "p95": 0.29, + "p99": 0.29 + }, + "ttfbMs": { + "p50": 0.24, + "p95": 0.26, + "p99": 0.26 + }, + "statuses": { + "200": 3 + } + }, + { + "name": "jobs create", + "method": "POST", + "path": "/api/jobs", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 2482.67, + "peakRps": 3, + "latencyMs": { + "p50": 0.36, + "p95": 0.48, + "p99": 0.48 + }, + "ttfbMs": { + "p50": 0.33, + "p95": 0.45, + "p99": 0.45 + }, + "statuses": { + "201": 3 + } + }, + { + "name": "proposals list", + "method": "GET", + "path": "/api/proposals", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 3000, + "peakRps": 3, + "latencyMs": { + "p50": 0.19, + "p95": 0.23, + "p99": 0.23 + }, + "ttfbMs": { + "p50": 0.17, + "p95": 0.21, + "p99": 0.21 + }, + "statuses": { + "200": 3 + } + }, + { + "name": "proposals create", + "method": "POST", + "path": "/api/proposals", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 2769.12, + "peakRps": 3, + "latencyMs": { + "p50": 0.36, + "p95": 0.39, + "p99": 0.39 + }, + "ttfbMs": { + "p50": 0.32, + "p95": 0.37, + "p99": 0.37 + }, + "statuses": { + "201": 3 + } + }, + { + "name": "payments create", + "method": "POST", + "path": "/api/payments", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 3000, + "peakRps": 3, + "latencyMs": { + "p50": 0.31, + "p95": 0.33, + "p99": 0.33 + }, + "ttfbMs": { + "p50": 0.28, + "p95": 0.31, + "p99": 0.31 + }, + "statuses": { + "201": 3 + } + }, + { + "name": "reviews list", + "method": "GET", + "path": "/api/reviews", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 3000, + "peakRps": 3, + "latencyMs": { + "p50": 0.2, + "p95": 0.25, + "p99": 0.25 + }, + "ttfbMs": { + "p50": 0.19, + "p95": 0.23, + "p99": 0.23 + }, + "statuses": { + "200": 3 + } + }, + { + "name": "reviews create", + "method": "POST", + "path": "/api/reviews", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 3000, + "peakRps": 3, + "latencyMs": { + "p50": 0.25, + "p95": 0.25, + "p99": 0.25 + }, + "ttfbMs": { + "p50": 0.23, + "p95": 0.23, + "p99": 0.23 + }, + "statuses": { + "201": 3 + } + }, + { + "name": "messages list", + "method": "GET", + "path": "/api/messages", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 3000, + "peakRps": 3, + "latencyMs": { + "p50": 0.2, + "p95": 0.21, + "p99": 0.21 + }, + "ttfbMs": { + "p50": 0.19, + "p95": 0.2, + "p99": 0.2 + }, + "statuses": { + "200": 3 + } + }, + { + "name": "messages create", + "method": "POST", + "path": "/api/messages", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 2215.25, + "peakRps": 3, + "latencyMs": { + "p50": 0.41, + "p95": 0.63, + "p99": 0.63 + }, + "ttfbMs": { + "p50": 0.4, + "p95": 0.61, + "p99": 0.61 + }, + "statuses": { + "201": 3 + } + }, + { + "name": "notifications list", + "method": "GET", + "path": "/api/notifications", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 3000, + "peakRps": 3, + "latencyMs": { + "p50": 0.22, + "p95": 0.22, + "p99": 0.22 + }, + "ttfbMs": { + "p50": 0.21, + "p95": 0.21, + "p99": 0.21 + }, + "statuses": { + "200": 3 + } + }, + { + "name": "notifications create", + "method": "POST", + "path": "/api/notifications", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 3000, + "peakRps": 3, + "latencyMs": { + "p50": 0.26, + "p95": 0.26, + "p99": 0.26 + }, + "ttfbMs": { + "p50": 0.24, + "p95": 0.24, + "p99": 0.24 + }, + "statuses": { + "201": 3 + } + }, + { + "name": "uploads create", + "method": "POST", + "path": "/api/uploads", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 931.33, + "peakRps": 3, + "latencyMs": { + "p50": 0.5, + "p95": 2.09, + "p99": 2.09 + }, + "ttfbMs": { + "p50": 0.48, + "p95": 2.07, + "p99": 2.07 + }, + "statuses": { + "201": 3 + } + }, + { + "name": "search query", + "method": "GET", + "path": "/api/search?q=freelance", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 3000, + "peakRps": 3, + "latencyMs": { + "p50": 0.24, + "p95": 0.52, + "p99": 0.52 + }, + "ttfbMs": { + "p50": 0.23, + "p95": 0.5, + "p99": 0.5 + }, + "statuses": { + "200": 3 + } + }, + { + "name": "admin metrics", + "method": "GET", + "path": "/api/admin/metrics", + "requests": 3, + "errors": 0, + "errorRatePct": 0, + "sustainedRps": 2732.86, + "peakRps": 3, + "latencyMs": { + "p50": 0.28, + "p95": 0.55, + "p99": 0.55 + }, + "ttfbMs": { + "p50": 0.26, + "p95": 0.53, + "p99": 0.53 + }, + "statuses": { + "200": 3 + } + } + ], + "gates": [ + { + "name": "health", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 15.13, + "errorRatePct": 0 + }, + { + "name": "auth register", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 5.54, + "errorRatePct": 0 + }, + { + "name": "auth login", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.58, + "errorRatePct": 0 + }, + { + "name": "auth oauth callback", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.53, + "errorRatePct": 0 + }, + { + "name": "auth refresh", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.4, + "errorRatePct": 0 + }, + { + "name": "users list", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.31, + "errorRatePct": 0 + }, + { + "name": "users create", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.45, + "errorRatePct": 0 + }, + { + "name": "jobs list", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.29, + "errorRatePct": 0 + }, + { + "name": "jobs create", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.48, + "errorRatePct": 0 + }, + { + "name": "proposals list", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.23, + "errorRatePct": 0 + }, + { + "name": "proposals create", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.39, + "errorRatePct": 0 + }, + { + "name": "payments create", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.33, + "errorRatePct": 0 + }, + { + "name": "reviews list", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.25, + "errorRatePct": 0 + }, + { + "name": "reviews create", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.25, + "errorRatePct": 0 + }, + { + "name": "messages list", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.21, + "errorRatePct": 0 + }, + { + "name": "messages create", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.63, + "errorRatePct": 0 + }, + { + "name": "notifications list", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.22, + "errorRatePct": 0 + }, + { + "name": "notifications create", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.26, + "errorRatePct": 0 + }, + { + "name": "uploads create", + "passed": true, + "threshold": { + "p99LatencyMs": 750, + "errorRatePct": 0 + }, + "p99LatencyMs": 2.09, + "errorRatePct": 0 + }, + { + "name": "search query", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.52, + "errorRatePct": 0 + }, + { + "name": "admin metrics", + "passed": true, + "threshold": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "p99LatencyMs": 0.55, + "errorRatePct": 0 + } + ] +} diff --git a/benchmarks/results/latest.md b/benchmarks/results/latest.md new file mode 100644 index 000000000..ef34acd17 --- /dev/null +++ b/benchmarks/results/latest.md @@ -0,0 +1,31 @@ +# API Benchmark Summary + +Mode: smoke +Target: local Express app +Generated: 2026-05-27T08:23:05.133Z +Iterations per endpoint: 3 +Concurrency per endpoint: 1 + +| Endpoint | Method | Requests | Error % | Sustained RPS | p50 | p95 | p99 | TTFB p95 | Gate | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | +| health | GET /health | 3 | 0 | 174.67 | 1.35 ms | 15.13 ms | 15.13 ms | 14.67 ms | pass | +| auth register | POST /api/auth/register | 3 | 0 | 425.88 | 0.84 ms | 5.54 ms | 5.54 ms | 5.48 ms | pass | +| auth login | POST /api/auth/login | 3 | 0 | 1850.57 | 0.51 ms | 0.58 ms | 0.58 ms | 0.54 ms | pass | +| auth oauth callback | GET /api/auth/oauth/github/callback | 3 | 0 | 2566.93 | 0.34 ms | 0.53 ms | 0.53 ms | 0.5 ms | pass | +| auth refresh | POST /api/auth/refresh | 3 | 0 | 2546.33 | 0.4 ms | 0.4 ms | 0.4 ms | 0.37 ms | pass | +| users list | GET /api/users | 3 | 0 | 3000 | 0.3 ms | 0.31 ms | 0.31 ms | 0.29 ms | pass | +| users create | POST /api/users | 3 | 0 | 2578.89 | 0.37 ms | 0.45 ms | 0.45 ms | 0.42 ms | pass | +| jobs list | GET /api/jobs | 3 | 0 | 3000 | 0.27 ms | 0.29 ms | 0.29 ms | 0.26 ms | pass | +| jobs create | POST /api/jobs | 3 | 0 | 2482.67 | 0.36 ms | 0.48 ms | 0.48 ms | 0.45 ms | pass | +| proposals list | GET /api/proposals | 3 | 0 | 3000 | 0.19 ms | 0.23 ms | 0.23 ms | 0.21 ms | pass | +| proposals create | POST /api/proposals | 3 | 0 | 2769.12 | 0.36 ms | 0.39 ms | 0.39 ms | 0.37 ms | pass | +| payments create | POST /api/payments | 3 | 0 | 3000 | 0.31 ms | 0.33 ms | 0.33 ms | 0.31 ms | pass | +| reviews list | GET /api/reviews | 3 | 0 | 3000 | 0.2 ms | 0.25 ms | 0.25 ms | 0.23 ms | pass | +| reviews create | POST /api/reviews | 3 | 0 | 3000 | 0.25 ms | 0.25 ms | 0.25 ms | 0.23 ms | pass | +| messages list | GET /api/messages | 3 | 0 | 3000 | 0.2 ms | 0.21 ms | 0.21 ms | 0.2 ms | pass | +| messages create | POST /api/messages | 3 | 0 | 2215.25 | 0.41 ms | 0.63 ms | 0.63 ms | 0.61 ms | pass | +| notifications list | GET /api/notifications | 3 | 0 | 3000 | 0.22 ms | 0.22 ms | 0.22 ms | 0.21 ms | pass | +| notifications create | POST /api/notifications | 3 | 0 | 3000 | 0.26 ms | 0.26 ms | 0.26 ms | 0.24 ms | pass | +| uploads create | POST /api/uploads | 3 | 0 | 931.33 | 0.5 ms | 2.09 ms | 2.09 ms | 2.07 ms | pass | +| search query | GET /api/search?q=freelance | 3 | 0 | 3000 | 0.24 ms | 0.52 ms | 0.52 ms | 0.5 ms | pass | +| admin metrics | GET /api/admin/metrics | 3 | 0 | 2732.86 | 0.28 ms | 0.55 ms | 0.55 ms | 0.53 ms | pass | diff --git a/benchmarks/run-api-benchmarks.mjs b/benchmarks/run-api-benchmarks.mjs new file mode 100644 index 000000000..5248e8129 --- /dev/null +++ b/benchmarks/run-api-benchmarks.mjs @@ -0,0 +1,253 @@ +#!/usr/bin/env node +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { performance } from "node:perf_hooks"; +import { createApp } from "../apps/api/src/app.js"; +import { signAccessToken } from "../apps/api/src/utils/jwt.js"; +import { createScenarios } from "./scenarios.mjs"; + +const ROOT = new URL("..", import.meta.url); +const RESULTS_DIR = new URL("benchmarks/results/", ROOT); +const THRESHOLDS_FILE = new URL("benchmarks/thresholds.json", ROOT); +const isSmoke = process.argv.includes("--smoke") || process.env.BENCHMARK_SMOKE === "1"; + +function readNumber(name, fallback) { + const value = Number(process.env[name]); + return Number.isFinite(value) && value > 0 ? value : fallback; +} + +function readNonNegativeNumber(name, fallback) { + const value = Number(process.env[name]); + return Number.isFinite(value) && value >= 0 ? value : fallback; +} + +function percentile(values, pct) { + if (values.length === 0) { + return 0; + } + + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil((pct / 100) * sorted.length) - 1); + return sorted[index]; +} + +function round(value, digits = 2) { + return Number(value.toFixed(digits)); +} + +async function startLocalServer() { + const app = createApp(); + const server = app.listen(0, "127.0.0.1"); + + await new Promise((resolve, reject) => { + server.once("listening", resolve); + server.once("error", reject); + }); + + const { port } = server.address(); + return { + baseUrl: `http://127.0.0.1:${port}`, + close: () => new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))) + }; +} + +async function requestOnce(baseUrl, scenario, index) { + const url = new URL(scenario.path, baseUrl); + const headers = { ...(scenario.headers ?? {}) }; + const init = { method: scenario.method, headers }; + if (scenario.json) { + init.body = JSON.stringify(scenario.json(index)); + } + if (scenario.formData) { + init.body = scenario.formData(index); + } + + const start = performance.now(); + let response; + let ttfbMs; + let body = ""; + let error; + + try { + response = await fetch(url, init); + ttfbMs = performance.now() - start; + body = await response.text(); + } catch (caught) { + error = caught; + ttfbMs = performance.now() - start; + } + + const latencyMs = performance.now() - start; + const status = response?.status ?? 0; + const ok = Boolean(response?.ok); + + return { + status, + ok, + startedAtMs: start, + latencyMs, + ttfbMs, + bytes: Buffer.byteLength(body), + error: error?.message + }; +} + +async function runScenario(baseUrl, scenario, options) { + const total = options.iterations; + const concurrency = Math.min(options.concurrency, total); + let cursor = 0; + const results = []; + const startedAt = performance.now(); + + async function worker() { + while (cursor < total) { + const current = cursor; + cursor += 1; + results.push(await requestOnce(baseUrl, scenario, current)); + } + } + + await Promise.all(Array.from({ length: concurrency }, worker)); + const durationMs = performance.now() - startedAt; + const latencies = results.map((result) => result.latencyMs); + const ttfb = results.map((result) => result.ttfbMs); + const errors = results.filter((result) => !result.ok).length; + + return { + name: scenario.name, + method: scenario.method, + path: scenario.path, + requests: results.length, + errors, + errorRatePct: round((errors / Math.max(results.length, 1)) * 100), + sustainedRps: round(results.length / Math.max(durationMs / 1000, 0.001)), + peakRps: round(Math.max(...chunkedRps(results), 0)), + latencyMs: { + p50: round(percentile(latencies, 50)), + p95: round(percentile(latencies, 95)), + p99: round(percentile(latencies, 99)) + }, + ttfbMs: { + p50: round(percentile(ttfb, 50)), + p95: round(percentile(ttfb, 95)), + p99: round(percentile(ttfb, 99)) + }, + statuses: summarizeStatuses(results), + sampleError: results.find((result) => result.error)?.error + }; +} + +function chunkedRps(results) { + const firstStart = Math.min(...results.map((result) => result.startedAtMs)); + const buckets = new Map(); + for (const result of results) { + const bucket = Math.floor((result.startedAtMs - firstStart) / 1000); + buckets.set(bucket, (buckets.get(bucket) ?? 0) + 1); + } + return [...buckets.values()]; +} + +function summarizeStatuses(results) { + const counts = {}; + for (const result of results) { + counts[result.status] = (counts[result.status] ?? 0) + 1; + } + return counts; +} + +function thresholdFor(thresholds, name) { + return { + ...thresholds.defaults, + ...(thresholds.endpoints?.[name] ?? {}) + }; +} + +function evaluateThresholds(summaries, thresholds) { + return summaries.map((summary) => { + const threshold = thresholdFor(thresholds, summary.name); + return { + name: summary.name, + passed: + summary.latencyMs.p99 <= threshold.p99LatencyMs && + summary.errorRatePct <= threshold.errorRatePct, + threshold, + p99LatencyMs: summary.latencyMs.p99, + errorRatePct: summary.errorRatePct + }; + }); +} + +function renderMarkdown(report) { + const lines = [ + "# API Benchmark Summary", + "", + `Mode: ${report.mode}`, + `Target: ${report.target}`, + `Generated: ${report.generatedAt}`, + `Iterations per endpoint: ${report.options.iterations}`, + `Concurrency per endpoint: ${report.options.concurrency}`, + "", + "| Endpoint | Method | Requests | Error % | Sustained RPS | p50 | p95 | p99 | TTFB p95 | Gate |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |" + ]; + + for (const summary of report.summaries) { + const gate = report.gates.find((item) => item.name === summary.name); + lines.push( + `| ${summary.name} | ${summary.method} ${summary.path} | ${summary.requests} | ${summary.errorRatePct} | ${summary.sustainedRps} | ${summary.latencyMs.p50} ms | ${summary.latencyMs.p95} ms | ${summary.latencyMs.p99} ms | ${summary.ttfbMs.p95} ms | ${gate?.passed ? "pass" : "fail"} |` + ); + } + + return `${lines.join("\n")}\n`; +} + +async function main() { + const localServer = process.env.BENCHMARK_TARGET_URL ? null : await startLocalServer(); + const baseUrl = process.env.BENCHMARK_TARGET_URL ?? localServer.baseUrl; + const authToken = process.env.BENCHMARK_AUTH_TOKEN ?? signAccessToken({ sub: "usr_benchmark_admin", role: "admin" }); + const options = { + iterations: isSmoke ? 3 : readNumber("BENCHMARK_ITERATIONS", 20), + concurrency: isSmoke ? 1 : readNumber("BENCHMARK_CONCURRENCY", 4) + }; + + try { + const scenarios = createScenarios({ authToken }); + const thresholds = JSON.parse(await readFile(THRESHOLDS_FILE, "utf8")); + thresholds.defaults.p99LatencyMs = readNumber("BENCHMARK_P99_THRESHOLD_MS", thresholds.defaults.p99LatencyMs); + thresholds.defaults.errorRatePct = readNonNegativeNumber( + "BENCHMARK_ERROR_RATE_THRESHOLD", + thresholds.defaults.errorRatePct + ); + const summaries = []; + + for (const scenario of scenarios) { + summaries.push(await runScenario(baseUrl, scenario, options)); + } + + const gates = evaluateThresholds(summaries, thresholds); + const report = { + generatedAt: new Date().toISOString(), + mode: isSmoke ? "smoke" : "full", + target: process.env.BENCHMARK_TARGET_URL ? baseUrl : "local Express app", + options, + summaries, + gates + }; + + await mkdir(RESULTS_DIR, { recursive: true }); + await writeFile(new URL("latest.json", RESULTS_DIR), `${JSON.stringify(report, null, 2)}\n`); + await writeFile(new URL("latest.md", RESULTS_DIR), renderMarkdown(report)); + + const failed = gates.filter((gate) => !gate.passed); + if (failed.length > 0) { + console.error(`Benchmark threshold failures: ${failed.map((gate) => gate.name).join(", ")}`); + process.exitCode = 1; + } + } finally { + await localServer?.close(); + } +} + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/benchmarks/scenarios.mjs b/benchmarks/scenarios.mjs new file mode 100644 index 000000000..52067bd9e --- /dev/null +++ b/benchmarks/scenarios.mjs @@ -0,0 +1,102 @@ +export function createScenarios({ authToken }) { + const json = { "content-type": "application/json" }; + const auth = { authorization: `Bearer ${authToken}` }; + + return [ + { name: "health", method: "GET", path: "/health" }, + { + name: "auth register", + method: "POST", + path: "/api/auth/register", + headers: json, + json: (i) => ({ + email: `bench-register-${i}@example.com`, + password: "benchmark-password", + role: "client" + }) + }, + { + name: "auth login", + method: "POST", + path: "/api/auth/login", + headers: json, + json: () => ({ email: "bench-login@example.com", password: "benchmark-password" }) + }, + { name: "auth oauth callback", method: "GET", path: "/api/auth/oauth/github/callback" }, + { name: "auth refresh", method: "POST", path: "/api/auth/refresh", headers: json, json: () => ({}) }, + { name: "users list", method: "GET", path: "/api/users" }, + { + name: "users create", + method: "POST", + path: "/api/users", + headers: json, + json: (i) => ({ email: `bench-user-${i}@example.com`, role: "freelancer", status: "active" }) + }, + { name: "jobs list", method: "GET", path: "/api/jobs" }, + { + name: "jobs create", + method: "POST", + path: "/api/jobs", + headers: json, + json: (i) => ({ + title: `Benchmark job ${i}`, + description: "Synthetic benchmark job payload for API load testing.", + budgetMin: 500, + budgetMax: 2500, + categoryId: "cat_web", + skills: ["node", "api", "testing"] + }) + }, + { name: "proposals list", method: "GET", path: "/api/proposals" }, + { + name: "proposals create", + method: "POST", + path: "/api/proposals", + headers: json, + json: (i) => ({ jobId: `job_${i}`, freelancerId: "usr_benchmark", bidAmount: 1200, coverLetter: "Benchmark proposal" }) + }, + { + name: "payments create", + method: "POST", + path: "/api/payments", + headers: json, + json: () => ({ amount: 1299, currency: "usd", metadata: { source: "benchmark" } }) + }, + { name: "reviews list", method: "GET", path: "/api/reviews" }, + { + name: "reviews create", + method: "POST", + path: "/api/reviews", + headers: json, + json: (i) => ({ jobId: `job_${i}`, rating: 5, comment: "Benchmark review" }) + }, + { name: "messages list", method: "GET", path: "/api/messages" }, + { + name: "messages create", + method: "POST", + path: "/api/messages", + headers: json, + json: (i) => ({ conversationId: `conv_${i}`, senderId: "usr_benchmark", body: "Benchmark message body" }) + }, + { name: "notifications list", method: "GET", path: "/api/notifications" }, + { + name: "notifications create", + method: "POST", + path: "/api/notifications", + headers: json, + json: (i) => ({ userId: `usr_${i}`, type: "benchmark", message: "Benchmark notification" }) + }, + { + name: "uploads create", + method: "POST", + path: "/api/uploads", + formData: () => { + const body = new FormData(); + body.append("file", new Blob(["benchmark upload payload"], { type: "text/plain" }), "benchmark.txt"); + return body; + } + }, + { name: "search query", method: "GET", path: "/api/search?q=freelance" }, + { name: "admin metrics", method: "GET", path: "/api/admin/metrics", headers: auth } + ]; +} diff --git a/benchmarks/thresholds.json b/benchmarks/thresholds.json new file mode 100644 index 000000000..105ff4713 --- /dev/null +++ b/benchmarks/thresholds.json @@ -0,0 +1,12 @@ +{ + "defaults": { + "p99LatencyMs": 500, + "errorRatePct": 0 + }, + "endpoints": { + "uploads create": { + "p99LatencyMs": 750, + "errorRatePct": 0 + } + } +} diff --git a/demos/api-benchmark-smoke-demo.mp4 b/demos/api-benchmark-smoke-demo.mp4 new file mode 100644 index 000000000..cefaca0d7 Binary files /dev/null and b/demos/api-benchmark-smoke-demo.mp4 differ diff --git a/demos/api-benchmark-smoke-demo.svg b/demos/api-benchmark-smoke-demo.svg new file mode 100644 index 000000000..89ae1be82 --- /dev/null +++ b/demos/api-benchmark-smoke-demo.svg @@ -0,0 +1,11 @@ + + + + API benchmark smoke + 21 API endpoints covered + npm run benchmark:smoke + 0% error rate + p50, p95, p99, RPS, and TTFB captured + GitHub Actions smoke gate passing + SecureBananaLabs bug-bounty #30 + diff --git a/demos/api-benchmark-smoke-demo.svg.png b/demos/api-benchmark-smoke-demo.svg.png new file mode 100644 index 000000000..78e9ffd47 Binary files /dev/null and b/demos/api-benchmark-smoke-demo.svg.png differ diff --git a/package.json b/package.json index 675e6e69d..0cff9eb39 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,8 @@ ], "scripts": { "build": "echo \"Run package-specific builds (e.g. npm run build -w apps/web)\"", + "benchmark": "node benchmarks/run-api-benchmarks.mjs", + "benchmark:smoke": "node benchmarks/run-api-benchmarks.mjs --smoke", "lint": "echo \"No root lint configured\"", "test": "npm run test -w apps/api" }