Configurable batch size #389
+201
−123
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -39,11 +39,23 @@ | |
|
|
||
|
|
||
|
|
||
| async function main() { | ||
| // Run performance tests and store outputs in memory | ||
| const messageCount = process.env.MESSAGE_COUNT ? +process.env.MESSAGE_COUNT : 50000; | ||
| let skipCtpTest = process.env.SKIP_CTP_TEST ? process.env.SKIP_CTP_TEST === 'true' : false; | ||
| const concurrentRun = process.env.CONCURRENT_RUN ? process.env.CONCURRENT_RUN === 'true' : false; | ||
| if (concurrentRun) { | ||
| skipCtpTest = true; | ||
| } | ||
| if (!process.env.CONSUMER_MAX_BATCH_SIZE) { | ||
| process.env.CONSUMER_MAX_BATCH_SIZE = '-1'; | ||
| } | ||
| if (!process.env.PARTITIONS_CONSUMED_CONCURRENTLY) { | ||
| process.env.PARTITIONS_CONSUMED_CONCURRENTLY = '2'; | ||
| } | ||
| if (!process.env.COMPRESSION) { | ||
| process.env.COMPRESSION = 'GZIP'; | ||
| } | ||
| const consumerMode = process.env.CONSUMER_MODE || 'all'; | ||
| const produceToSecondTopic = process.env.PRODUCE_TO_SECOND_TOPIC ? process.env.PRODUCE_TO_SECOND_TOPIC === 'true' : false; | ||
| const produceToSecondTopicParam = produceToSecondTopic ? '--produce-to-second-topic' : ''; | ||
|
|
@@ -58,10 +70,10 @@ | |
| } | ||
| let outputConfluentProducerConsumer; | ||
| let outputKjsProducerConsumer; | ||
| const groupIdEachMessageConfluent = `test-group-confluent-message-` + Math.random(); | ||
| const groupIdEachBatchConfluent = `test-group-confluent-batch-` + Math.random(); | ||
| const groupIdEachMessageKafkaJS = `test-group-kafkajs-message-` + Math.random(); | ||
| const groupIdEachBatchKafkaJS = `test-group-kafkajs-batch-` + Math.random(); | ||
| if (consumerModeAll || consumerModeEachMessage) { | ||
| console.log(`Confluent eachMessage group id: ${groupIdEachMessageConfluent}`); | ||
| console.log(`KafkaJS eachMessage group id: ${groupIdEachMessageKafkaJS}`); | ||
|
|
@@ -119,11 +131,11 @@ | |
|
|
||
| console.log('Running Confluent CTP test...'); | ||
| const outputConfluentCtp = skipCtpTest ? '' : | ||
| (await runCommand('MODE=confluent MESSAGE_COUNT=5000 node performance-consolidated.js --create-topics --ctp')); | ||
|
|
||
| console.log('Running KafkaJS CTP test...'); | ||
| const outputKjsCtp = skipCtpTest ? '' : | ||
| (await runCommand('MODE=kafkajs MESSAGE_COUNT=5000 node performance-consolidated.js --create-topics --ctp')); | ||
|
|
||
| // Extract Confluent results | ||
| let ctpConfluent, ctpKjs; | ||
|
|
@@ -230,22 +242,22 @@ | |
| consumerKjsMessageRate = extractValue(outputKjsProducerConsumer, '=== Consumer Rate msg/s (eachMessage):'); | ||
| consumerKjsMessageAvgLatencyT0T1 = extractValue(outputKjsProducerConsumer, '=== Consumer average E2E latency T0-T1 (eachMessage):'); | ||
| consumerKjsMessageMaxLatencyT0T1 = extractValue(outputKjsProducerConsumer, '=== Consumer max E2E latency T0-T1 (eachMessage):'); | ||
| consumerKjsMessageAvgLatencyT0T2 = extractValue(outputKjsProducerConsumer, '=== Consumer average E2E latency T0-T2 (eachMessage):'); | ||
|
Check failure on line 245 in ci/tests/run_perf_test.js
|
||
| consumerKjsMessageMaxLatencyT0T2 = extractValue(outputKjsProducerConsumer, '=== Consumer max E2E latency T0-T2 (eachMessage):'); | ||
|
Check failure on line 246 in ci/tests/run_perf_test.js
|
||
| consumerKjsTime = extractValue(outputKjsProducerConsumer, '=== Consumption time (eachMessage):'); | ||
| consumerKjsMessageAverageRSS = extractValue(outputKjsProducerConsumer, '=== Average consumer-each-message RSS KB:'); | ||
| consumerKjsMessageMaxRSS = extractValue(outputKjsProducerConsumer, '=== Max consumer-each-message RSS KB:'); | ||
| consumerKjsMessageAverageBrokerLag = extractValue(outputKjsProducerConsumer, `=== Average broker lag (${groupIdEachMessageKafkaJS}):`); | ||
| consumerKjsMessageMaxBrokerLag = extractValue(outputKjsProducerConsumer, `=== Max broker lag (${groupIdEachMessageKafkaJS}):`); | ||
| consumerKjsMessageTotalLagMeasurements = extractValue(outputKjsProducerConsumer, `=== Sample size for broker lag measurement (${groupIdEachMessageKafkaJS}):`); | ||
| } | ||
| if (consumerModeAll || consumerModeEachBatch) { | ||
| consumerKjsBatch = extractValue(outputKjsProducerConsumer, '=== Consumer Rate MB/s (eachBatch):'); | ||
| consumerKjsBatchRate = extractValue(outputKjsProducerConsumer, '=== Consumer Rate msg/s (eachBatch):'); | ||
| consumerKjsBatchAvgLatencyT0T1 = extractValue(outputKjsProducerConsumer, '=== Consumer average E2E latency T0-T1 (eachBatch):'); | ||
| consumerKjsBatchMaxLatencyT0T1 = extractValue(outputKjsProducerConsumer, '=== Consumer max E2E latency T0-T1 (eachBatch):'); | ||
| consumerKjsBatchAvgLatencyT0T2 = extractValue(outputKjsProducerConsumer, '=== Consumer average E2E latency T0-T2 (eachBatch):'); | ||
|
Check failure on line 259 in ci/tests/run_perf_test.js
|
||
| consumerKjsBatchMaxLatencyT0T2 = extractValue(outputKjsProducerConsumer, '=== Consumer max E2E latency T0-T2 (eachBatch):'); | ||
|
Check failure on line 260 in ci/tests/run_perf_test.js
|
||
| consumerKjsBatchTime = extractValue(outputKjsProducerConsumer, '=== Consumption time (eachBatch):'); | ||
| consumerKjsBatchAverageLag = extractValue(outputKjsProducerConsumer, '=== Average eachBatch lag:'); | ||
| consumerKjsBatchMaxLag = extractValue(outputKjsProducerConsumer, '=== Max eachBatch lag:'); | ||
|
|
@@ -269,37 +281,41 @@ | |
| if (consumerModeAll || consumerModeEachMessage) { | ||
| console.log(`Consumer rates MB/s (eachMessage): confluent ${consumerConfluentMessage}, kafkajs ${consumerKjsMessage}`); | ||
| console.log(`Consumer rates msg/s (eachMessage): confluent ${consumerConfluentMessageRate}, kafkajs ${consumerKjsMessageRate}`); | ||
| console.log(`Consumption time (eachMessage): confluent ${consumerConfluentTime}, kafkajs ${consumerKjsTime}`); | ||
| console.log(`Average RSS (eachMessage): confluent ${consumerConfluentMessageAverageRSS}, kafkajs ${consumerKjsMessageAverageRSS}`); | ||
| console.log(`Max RSS (eachMessage): confluent ${consumerConfluentMessageMaxRSS}, kafkajs ${consumerKjsMessageMaxRSS}`); | ||
| if (concurrentRun) { | ||
| console.log(`Consumer average E2E latency T0-T1 (eachMessage): confluent ${consumerConfluentMessageAvgLatencyT0T1}, kafkajs ${consumerKjsMessageAvgLatencyT0T1}`); | ||
| console.log(`Consumer max E2E latency T0-T1 (eachMessage): confluent ${consumerConfluentMessageMaxLatencyT0T1}, kafkajs ${consumerKjsMessageMaxLatencyT0T1}`); | ||
| if (produceToSecondTopic) { | ||
| console.log(`Consumer average E2E latency T0-T2 (eachMessage): confluent ${consumerConfluentMessageAvgLatencyT0T2}, kafkajs ${consumerKjsMessageAvgLatencyT0T2}`); | ||
| console.log(`Consumer max E2E latency T0-T2 (eachMessage): confluent ${consumerConfluentMessageMaxLatencyT0T2}, kafkajs ${consumerKjsMessageMaxLatencyT0T2}`); | ||
| } | ||
| console.log(`Average broker lag (eachMessage): confluent ${consumerConfluentMessageAverageBrokerLag}, kafkajs ${consumerKjsMessageAverageBrokerLag}`); | ||
| console.log(`Max broker lag (eachMessage): confluent ${consumerConfluentMessageMaxBrokerLag}, kafkajs ${consumerKjsMessageMaxBrokerLag}`); | ||
| console.log(`Sample size for broker lag measurement (eachMessage): confluent ${consumerConfluentMessageTotalLagMeasurements}, kafkajs ${consumerKjsMessageTotalLagMeasurements}`); | ||
| } | ||
| } | ||
| if (consumerModeAll || consumerModeEachBatch) { | ||
| console.log(`Consumer rates MB/s (eachBatch): confluent ${consumerConfluentBatch}, kafkajs ${consumerKjsBatch}`); | ||
| console.log(`Consumer rates msg/s (eachBatch): confluent ${consumerConfluentBatchRate}, kafkajs ${consumerKjsBatchRate}`); | ||
| console.log(`Consumption time (eachBatch): confluent ${consumerConfluentBatchTime}, kafkajs ${consumerKjsBatchTime}`); | ||
| console.log(`Average eachBatch size: confluent ${consumerConfluentBatchAverageSize}, kafkajs ${consumerKjsBatchAverageSize}`); | ||
| console.log(`Average RSS (eachBatch): confluent ${consumerConfluentBatchAverageRSS}, kafkajs ${consumerKjsBatchAverageRSS}`); | ||
| console.log(`Max RSS (eachBatch): confluent ${consumerConfluentBatchMaxRSS}, kafkajs ${consumerKjsBatchMaxRSS}`); | ||
| if (concurrentRun) { | ||
| console.log(`Consumer average E2E latency T0-T1 (eachBatch): confluent ${consumerConfluentBatchAvgLatencyT0T1}, kafkajs ${consumerKjsBatchAvgLatencyT0T1}`); | ||
| console.log(`Consumer max E2E latency T0-T1 (eachBatch): confluent ${consumerConfluentBatchMaxLatencyT0T1}, kafkajs ${consumerKjsBatchMaxLatencyT0T1}`); | ||
| if (produceToSecondTopic) { | ||
| console.log(`Consumer average E2E latency T0-T2 (eachBatch): confluent ${consumerConfluentBatchAvgLatencyT0T2}, kafkajs ${consumerKjsBatchAvgLatencyT0T2}`); | ||
| console.log(`Consumer max E2E latency T0-T2 (eachBatch): confluent ${consumerConfluentBatchMaxLatencyT0T2}, kafkajs ${consumerKjsBatchMaxLatencyT0T2}`); | ||
| } | ||
| console.log(`Average eachBatch lag: confluent ${consumerConfluentBatchAverageLag}, kafkajs ${consumerKjsBatchAverageLag}`); | ||
| console.log(`Max eachBatch lag: confluent ${consumerConfluentBatchMaxLag}, kafkajs ${consumerKjsBatchMaxLag}`); | ||
| console.log(`Average broker lag (eachBatch): confluent ${consumerConfluentBatchAverageBrokerLag}, kafkajs ${consumerKjsBatchAverageBrokerLag}`); | ||
| console.log(`Max broker lag (eachBatch): confluent ${consumerConfluentBatchMaxBrokerLag}, kafkajs ${consumerKjsBatchMaxBrokerLag}`); | ||
| console.log(`Sample size for broker lag measurement (eachBatch): confluent ${consumerConfluentBatchTotalLagMeasurements}, kafkajs ${consumerKjsBatchTotalLagMeasurements}`); | ||
| } | ||
| } | ||
| if (!concurrentRun) { | ||
| console.log(`Average RSS: confluent ${consumerConfluentAverageRSS}, kafkajs ${consumerKjsAverageRSS}`); | ||
|
|
@@ -321,7 +337,7 @@ | |
| if (consumerModeAll || consumerModeEachMessage) { | ||
| if (belowThreshold(consumerConfluentMessage, consumerKjsMessage, maxPerformanceDifference)) { | ||
| console.log(`Consumer rates MB/s (eachMessage) differ by more than 30%: confluent ${consumerConfluentMessage}, kafkajs ${consumerKjsMessage}`); | ||
| // FIXME: improve consumer performance at least to KafkaJS level | ||
| errcode = 0; | ||
| } | ||
|
|
||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -89,7 +89,7 @@ function getPackageVersion(tag, branch) { | |
|
|
||
| // publish with a -devel suffix for EA and RC releases. | ||
| if (tag.prerelease.length > 0) { | ||
| baseVersion += '-' + tag.prerelease.join('.'); | ||
| } | ||
|
|
||
| console.log(`Package version is "${baseVersion}"`); | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -20,8 +20,8 @@ const saslPassword = process.env.SASL_PASSWORD; | |
| const topic = process.env.KAFKA_TOPIC || 'test-topic'; | ||
| const topic2 = process.env.KAFKA_TOPIC2 || 'test-topic2'; | ||
| const messageCount = process.env.MESSAGE_COUNT ? +process.env.MESSAGE_COUNT : 1000000; | ||
| const messageSize = process.env.MESSAGE_SIZE ? +process.env.MESSAGE_SIZE : 4096; | ||
| const batchSize = process.env.PRODUCER_BATCH_SIZE ? +process.env.PRODUCER_BATCH_SIZE : 100; | ||
| const compression = process.env.COMPRESSION || 'None'; | ||
| // Between 0 and 1, percentage of random bytes in each message | ||
| const randomness = process.env.RANDOMNESS ? +process.env.RANDOMNESS : 0.5; | ||
|
|
@@ -207,7 +207,9 @@ function logParameters(parameters) { | |
| console.log(` ProduceTopic: ${topic2}`); | ||
| console.log(` Message Count: ${messageCount}`); | ||
| // Seed the topic with messages | ||
| await runProducerCKJS(parameters, topic, batchSize, | ||
| warmupMessages, messageCount, messageSize, compression, | ||
| randomness, limitRPS); | ||
| startTrackingMemory(); | ||
| const ctpRate = await runConsumeTransformProduce(parameters, topic, topic2, warmupMessages, messageCount, messageProcessTimeMs, ctpConcurrency); | ||
| endTrackingMemory('consume-transform-produce', `consume-transform-produce-${mode}.json`); | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -14,7 +14,7 @@ else { | |
| } | ||
| } | ||
|
|
||
| function installHandlers(useTerminateTimeout) { | ||
| const handlers = { | ||
| terminationRequested: false, | ||
| terminateTimeout: null, | ||
|
|
@@ -26,8 +26,10 @@ function installHandlers() { | |
| process.on('SIGINT', terminationRequestedCallback); | ||
| process.on('SIGTERM', terminationRequestedCallback); | ||
| handlers.terminationRequestedCallback = terminationRequestedCallback; | ||
| if (useTerminateTimeout) { | ||
| handlers.terminateTimeout = setTimeout(terminationRequestedCallback, | ||
| TERMINATE_TIMEOUT_MS); | ||
| } | ||
| return handlers; | ||
| } | ||
|
|
||
|
|
@@ -57,7 +59,7 @@ function genericProduceToTopic(producer, topic, messages) { | |
| } | ||
|
|
||
| async function runConsumer(consumer, topic, warmupMessages, totalMessageCnt, eachBatch, partitionsConsumedConcurrently, stats, actionOnMessages) { | ||
| const handlers = installHandlers(totalMessageCnt === -1); | ||
| await consumer.connect(); | ||
| await consumer.subscribe({ topic }); | ||
|
|
||
|
|
@@ -179,18 +181,13 @@ async function runConsumer(consumer, topic, warmupMessages, totalMessageCnt, eac | |
| if (!startTime) { | ||
| startTime = hrtime.bigint(); | ||
| } else if (totalMessageCnt > 0 && messagesMeasured >= totalMessageCnt) { | ||
| stopConsuming(); | ||
| } | ||
| } | ||
|
|
||
| if (actionOnMessages) { | ||
| await actionOnMessages(batch.messages); | ||
| if (messagesMeasured > 0 && messages.length > 0) { | ||
| let i = 1; | ||
| const now = Date.now(); | ||
| for (const message of messages) { | ||
|
|
@@ -243,7 +240,7 @@ async function runConsumer(consumer, topic, warmupMessages, totalMessageCnt, eac | |
| } | ||
|
|
||
| async function runProducer(producer, topic, batchSize, warmupMessages, totalMessageCnt, msgSize, compression, randomness, limitRPS) { | ||
| const handlers = installHandlers(totalMessageCnt === -1); | ||
| let totalMessagesSent = 0; | ||
| let totalBytesSent = 0; | ||
|
|
||
|
|
@@ -347,7 +344,7 @@ async function runProducer(producer, topic, batchSize, warmupMessages, totalMess | |
| } | ||
|
|
||
| async function runLagMonitoring(admin, topic) { | ||
| const handlers = installHandlers(true); | ||
| let groupId = process.env.GROUPID_MONITOR; | ||
| if (!groupId) { | ||
| throw new Error("GROUPID_MONITOR environment variable not set"); | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -19,6 +19,9 @@ module.exports = { | |
| newCompatibleProducer, | ||
| }; | ||
|
|
||
|
|
||
| const CONSUMER_MAX_BATCH_SIZE = process.env.CONSUMER_MAX_BATCH_SIZE ? +process.env.CONSUMER_MAX_BATCH_SIZE : null; | ||
|
|
||
| function baseConfiguration(parameters) { | ||
| let ret = { | ||
| 'client.id': 'kafka-test-performance', | ||
|
|
@@ -146,6 +149,10 @@ function newCompatibleConsumer(parameters, eachBatch) { | |
| const autoCommitOpts = autoCommit > 0 ? | ||
| { 'enable.auto.commit': true, 'auto.commit.interval.ms': autoCommit } : | ||
| { 'enable.auto.commit': false }; | ||
| const jsOpts = {}; | ||
| if (eachBatch && CONSUMER_MAX_BATCH_SIZE !== null) { | ||
| jsOpts['js.consumer.max.batch.size'] = CONSUMER_MAX_BATCH_SIZE; | ||
| } | ||
|
|
||
| let groupId = eachBatch ? process.env.GROUPID_BATCH : process.env.GROUPID_MESSAGE; | ||
| if (!groupId) { | ||
|
|
@@ -157,6 +164,7 @@ function newCompatibleConsumer(parameters, eachBatch) { | |
| 'auto.offset.reset': 'earliest', | ||
| 'fetch.queue.backoff.ms': '100', | ||
| ...autoCommitOpts, | ||
| ...jsOpts, | ||
| }); | ||
| return new CompatibleConsumer(consumer); | ||
| } | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
nit: Let's make it 1.7.0