diff --git a/.changeset/jolly-canyons-glow.md b/.changeset/jolly-canyons-glow.md new file mode 100644 index 00000000..891fcf16 --- /dev/null +++ b/.changeset/jolly-canyons-glow.md @@ -0,0 +1,5 @@ +--- +"nostream": minor +--- + +perf: added k6 performance tests for connection and message rate limiting \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f787cf55..44340e50 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -302,6 +302,41 @@ To observe client and subscription counts in real-time during a test, you can in docker compose logs -f nostream ``` +## Performance Testing (k6) + +Nostream includes k6-based load tests to validate rate limiter behavior under concurrent WebSocket +connections. These tests verify that connection and message rate limits are correctly enforced. + +### Prerequisites + +Install [k6](https://grafana.com/docs/k6/latest/set-up/install-k6/) before running performance +tests. k6 is a standalone Go binary and is not included as an npm dependency. + +### Running the Tests + +Ensure the relay is running first (`pnpm run cli -- start`), then: + +```bash +# Test connection rate limiting +pnpm run cli -- dev test:perf:connection + +# Test message rate limiting +pnpm run cli -- dev test:perf:message +``` + +To test against a different relay instance: + +```bash +k6 run -e RELAY_URL=ws://your-host:8008 test/performance/connection-limiting-k6.ts +``` + +### What the Tests Validate + +- **Connection rate limiter** — Ramps concurrent connections through multiple stages and verifies + the relay rejects excess connections beyond the configured limit (default: 12 conn/sec). +- **Message rate limiter** — Opens WebSocket connections and sends continuous REQ messages, + verifying the relay returns NOTICE rejections when the message rate limit is exceeded. + ## Local Quality Checks Run dead code and dependency analysis before opening a pull request: diff --git a/package.json b/package.json index e062e8b0..dd74f3ab 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,8 @@ "test:load": "node -r ts-node/register ./scripts/security-load-test.ts", "smoke:nip03": "node -r ts-node/register scripts/smoke-nip03.ts", "test:integration": "cucumber-js", + "test:performance:connection-rate-limit": "k6 run test/performance/connection-limiting-k6.ts", + "test:performance:message-rate-limit": "k6 run test/performance/message-limiting-k6.ts", "cover:integration": "nyc --report-dir .coverage/integration pnpm run test:integration -p cover", "export": "node --env-file-if-exists=.env -r ts-node/register src/scripts/export-events.ts", "docker:compose:start": "pnpm run cli -- start", @@ -124,6 +126,7 @@ "@types/chai-as-promised": "^7.1.5", "@types/express": "4.17.21", "@types/js-yaml": "4.0.5", + "@types/k6": "^1.7.0", "@types/mocha": "^9.1.1", "@types/node": "^24.12.2", "@types/pg": "^8.6.5", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 042562e8..3fee7918 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -96,6 +96,9 @@ importers: '@types/js-yaml': specifier: 4.0.5 version: 4.0.5 + '@types/k6': + specifier: ^1.7.0 + version: 1.7.0 '@types/mocha': specifier: ^9.1.1 version: 9.1.1 @@ -763,6 +766,9 @@ packages: '@types/js-yaml@4.0.5': resolution: {integrity: sha512-FhpRzf927MNQdRZP0J5DLIdTXhjLYzeUTmLAu69mnVksLH9CJY3IuSeEgbKUki7GQZm0WqDkGzyxju2EZGD2wA==} + '@types/k6@1.7.0': + resolution: {integrity: sha512-oL4mckVcOPIA2HUrCVj3aQXCJgCqsQe35Uc4fRTffmrQuR24v92GJImnagqUaRnC1TQVJFx85o3aHQPP+0bxpg==} + '@types/minimist@1.2.5': resolution: {integrity: sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==} @@ -4289,6 +4295,8 @@ snapshots: '@types/js-yaml@4.0.5': {} + '@types/k6@1.7.0': {} + '@types/minimist@1.2.5': {} '@types/mocha@9.1.1': {} diff --git a/src/cli/commands/dev.ts b/src/cli/commands/dev.ts index 7b4fa3d0..59dbbbc9 100644 --- a/src/cli/commands/dev.ts +++ b/src/cli/commands/dev.ts @@ -137,3 +137,21 @@ export const runDevTestIntegration = async (): Promise => { () => runCommand('pnpm', ['run', 'test:integration']), ) } + +export const runDevTestPerfConnection = async (): Promise => { + return runWithSpinner( + 'Running connection rate limit performance test...', + 'Connection rate limit test completed', + 'Connection rate limit test failed', + () => runCommand('k6', ['run', 'test/performance/connection-limiting-k6.ts']), + ) +} + +export const runDevTestPerfMessage = async (): Promise => { + return runWithSpinner( + 'Running message rate limit performance test...', + 'Message rate limit test completed', + 'Message rate limit test failed', + () => runCommand('k6', ['run', 'test/performance/message-limiting-k6.ts']), + ) +} diff --git a/src/cli/index.ts b/src/cli/index.ts index 8366eabe..f3e4f53c 100644 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -28,6 +28,8 @@ import { runDevTestCli, runDevTestIntegration, runDevTestUnit, + runDevTestPerfConnection, + runDevTestPerfMessage } from './commands/dev' import { runTui } from './tui/main' import { logError, logInfo } from './utils/output' @@ -97,6 +99,8 @@ const devSubHelp: Record = { 'test:unit': 'Usage: nostream dev test:unit', 'test:cli': 'Usage: nostream dev test:cli', 'test:integration': 'Usage: nostream dev test:integration', + 'test:perf:connection': 'Usage: nostream dev test:perf:connection', + 'test:perf:message': 'Usage: nostream dev test:perf:message', } const withErrorBoundary = @@ -410,6 +414,10 @@ cli return runDevTestCli() case 'test:integration': return runDevTestIntegration() + case 'test:perf:connection': + return runDevTestPerfConnection() + case 'test:perf:message': + return runDevTestPerfMessage() default: logInfo( 'Usage: nostream dev [args]', diff --git a/test/performance/connection-limiting-k6.ts b/test/performance/connection-limiting-k6.ts new file mode 100644 index 00000000..2ee7272c --- /dev/null +++ b/test/performance/connection-limiting-k6.ts @@ -0,0 +1,83 @@ +import { check, sleep } from 'k6'; +import { Counter } from 'k6/metrics'; +import ws from 'k6/ws'; + +const relayUrl = __ENV.RELAY_URL || 'ws://127.0.0.1:8008'; +const connectionSuccess = new Counter('connection_success'); +const connectionRateLimited = new Counter('connection_rate_limited'); + +export const options = { + stages: [ + { duration: '10s', target: 3 }, + { duration: '10s', target: 6 }, + { duration: '10s', target: 12 }, + { duration: '10s', target: 18 }, + { duration: '5s', target: 0 }, + ], + thresholds: { + 'ws_connecting': ['p(95)<2000'], + }, +}; + +export default function () { + + const res = ws.connect(relayUrl, {}, function (socket) { + let intentionalClose = false + socket.on('close', () => { + if(!intentionalClose) { + connectionRateLimited.add(1); + } + }); + + socket.on('open', () => { + connectionSuccess.add(1); + }); + + socket.setTimeout(() => { + intentionalClose = true; + socket.close(); + }, 3000); + }); + + check(res, { + 'status is 101': (r) => r && r.status === 101, + }); + + sleep(0.5); +} + +export function handleSummary(data: any) { + const connSuccess = data.metrics?.connection_success?.values?.count || 0; + const connRateLimited = data.metrics?.connection_rate_limited?.values?.count || 0; + const iterations = data.metrics?.iterations?.values?.count || 0; + const checks = data.metrics?.checks?.values?.passes || 0; + const wsSessions = data.metrics?.ws_sessions?.values?.count || 0; + + const totalConnections = connSuccess + connRateLimited; + const successRate = totalConnections > 0 ? ((connSuccess / totalConnections) * 100).toFixed(2) : 0; + const rate = parseFloat(successRate as string); + const successStatus = rate >= 80 ? '✓ GOOD' : rate >= 50 ? '⚠ MODERATE' : '✗ POOR'; + + console.log(` + ╔════════════════════════════════════════════════════════════════╗ + ║ CONNECTION RATE LIMITER TEST RESULTS ║ + ╚════════════════════════════════════════════════════════════════╝ + + EXECUTION: + Iterations: ${iterations} + WebSocket Sessions: ${wsSessions} + Checks Passed: ${checks} + + CONNECTIONS: + ✓ Success (stayed open): ${connSuccess} + ✗ Rate Limited (closed): ${connRateLimited} + ───────────────────── + Total: ${totalConnections} + + PERFORMANCE: + Success Rate: ${successStatus} ${successRate}% + + ═══════════════════════════════════════════════════════════════════ + `); + return {}; +} \ No newline at end of file diff --git a/test/performance/message-limiting-k6.ts b/test/performance/message-limiting-k6.ts new file mode 100644 index 00000000..64ca8c4f --- /dev/null +++ b/test/performance/message-limiting-k6.ts @@ -0,0 +1,106 @@ +import { check } from 'k6'; +import { Counter } from 'k6/metrics'; +import ws from 'k6/ws'; + +const relayUrl = __ENV.RELAY_URL || 'ws://127.0.0.1:8008'; +const noticeCounter = new Counter('notice_messages'); +const eoseCounter = new Counter('eose_messages'); +const eventCounter = new Counter('event_messages'); +const errorCounter = new Counter('error_messages'); + +export const options = { + stages: [ + { duration: '10s', target: 1 }, + { duration: '10s', target: 2 }, + { duration: '10s', target: 4 }, + { duration: '5s', target: 0 }, + ], +}; + +export default function () { + const res = ws.connect(relayUrl, {}, function (socket) { + socket.on('open', function () { + let msgCount = 0; + socket.setInterval(function () { + msgCount++; + const text = JSON.stringify(['REQ', `sub-${Date.now()}-${msgCount}`, {limit: 10}]); + socket.send(text); + }, 1000); + }); + + socket.on('message', function (data) { + try { + const parsed = JSON.parse(data); + const msgType = parsed[0]; + + if (msgType === 'NOTICE') { + noticeCounter.add(1); + } else if (msgType === 'EOSE') { + eoseCounter.add(1); + } else if (msgType === 'EVENT') { + eventCounter.add(1); + } + } catch (e: any) { + errorCounter.add(1); + console.error('Failed to parse message:', e.message); + } + }); + + socket.setTimeout(function () { + socket.close(); + }, 9000); + }); + + check(res, { + 'status 101': (r) => r && r.status === 101, + }); +} + +export function handleSummary(data: any) { + const notices = data.metrics?.notice_messages?.values?.count || 0; + const eoses = data.metrics?.eose_messages?.values?.count || 0; + const events = data.metrics?.event_messages?.values?.count || 0; + const iterations = data.metrics?.iterations?.values?.count || 0; + const wsSessions = data.metrics?.ws_sessions?.values?.count || 0; + const msgsSent = data.metrics?.ws_msgs_sent?.values?.count || 0; + const msgsReceived = data.metrics?.ws_msgs_received?.values?.count || 0; + const dataReceived = data.metrics?.data_received?.values?.count || 0; + const checks = data.metrics?.checks?.values?.passes || 0; + + const totalMessages = notices + eoses + events; + const successRate = totalMessages > 0 ? ((eoses + events) / totalMessages * 100).toFixed(2) : 0; + + const rate = parseFloat(successRate as string); + const successStatus = rate >= 80 ? '✓ GOOD' : rate >= 50 ? '⚠ MODERATE' : '✗ POOR'; + const rateLimitStatus = notices > 0 ? '⚠ ACTIVE' : '✓ INACTIVE'; + + console.log(` +╔════════════════════════════════════════════════════════════════╗ +║ MESSAGE RATE LIMITER TEST RESULTS ║ +╚════════════════════════════════════════════════════════════════╝ + +EXECUTION: + Iterations: ${iterations} + WebSocket Sessions: ${wsSessions} + Checks Passed: ${checks} + +MESSAGES: + Sent: ${msgsSent} + Received: ${msgsReceived} + +MESSAGE TYPES: + ✗ NOTICE (rate limited): ${notices} + ✓ EOSE (query complete): ${eoses} + ◆ EVENT (results): ${events} + ───────────────────── + Total: ${totalMessages} + +PERFORMANCE: + Success Rate: ${successStatus} ${successRate}% + Data Received: ${dataReceived} bytes + Rate Limiter: ${rateLimitStatus} + +═══════════════════════════════════════════════════════════════════ + `); + return {}; +} \ No newline at end of file