YAR-14287: Change debug option to boolean (MaterializeInc#58)

* Change debug option to boolean

* Change dryRun to bool

* Fix merge conflicts

* Change clean option to bool

.github/workflows/integration.yaml

@@ -17,22 +17,22 @@ jobs:
         run: docker exec datagen datagen --version
 
       - name: Produce to Kafka from SQL Schema in JSON Format
-        run: docker exec datagen datagen -s /tests/schema.sql -f json -n 3 --record-size 100 -dr false -d true
+        run: docker exec datagen datagen -s /tests/schema.sql -f json -n 3 --record-size 100 -d
 
       - name: Produce to Kafka from JSON Schema in JSON Format
-        run: docker exec datagen datagen -s /tests/schema.json -f json -n 3 --record-size 100 -dr false -d true
+        run: docker exec datagen datagen -s /tests/schema.json -f json -n 3 --record-size 100 -d
 
       - name: Produce to Kafka from Avro Schema in JSON Format
-        run: docker exec datagen datagen -s /tests/schema.avsc -f json -n 3 --record-size 100 -dr false -d true
+        run: docker exec datagen datagen -s /tests/schema.avsc -f json -n 3 --record-size 100 -d
 
       - name: Produce to Kafka from SQL Schema in Avro Format
-        run: docker exec datagen datagen -s /tests/schema.sql -f avro -n 3 --record-size 100 -dr false -d true -w 100
+        run: docker exec datagen datagen -s /tests/schema.sql -f avro -n 3 --record-size 100 -d -w 100
 
       - name: Produce to Kafka from JSON Schema in Avro Format
-        run: docker exec datagen datagen -s /tests/schema.json -f avro -n 3 --record-size 100 -dr false -d true -w 100
+        run: docker exec datagen datagen -s /tests/schema.json -f avro -n 3 --record-size 100 -d -w 100
 
       - name: Produce to Kafka from Avro Schema in Avro Format
-        run: docker exec datagen datagen -s /tests/schema.avsc -f avro -n 3 --record-size 100 -dr false -d true -w 100
+        run: docker exec datagen datagen -s /tests/schema.avsc -f avro -n 3 --record-size 100 -d -w 100
 
       - name: Docker Compose Down
         run: docker compose down -v

CONTRIBUTING.md

@@ -8,12 +8,12 @@ Once you have made the changes to your local copy of the repository, you can run
 
 Also try to run the script with different options to make sure it works as expected, examples:
 
-    datagen -s ./tests/schema.json -f json -n 10 --record-size 100 -dr true -d true
-    datagen -s ./tests/schema.avsc -f json -n 10 --record-size 100 -dr true -d true
-    datagen -s ./tests/schema.sql -f json -n 10 --record-size 100 -dr true -d true
-    datagen -s ./tests/schema.json -f avro -n 10 --record-size 100 -dr true -d true
-    datagen -s ./tests/schema.avsc -f avro -n 10 --record-size 100 -dr true -d true
-    datagen -s ./tests/schema.sql -f avro -n 10 --record-size 100 -dr true -d true
+    datagen -s ./tests/schema.json -f json -n 10 --record-size 100 --dry-run --debug
+    datagen -s ./tests/schema.avsc -f json -n 10 --record-size 100 --dry-run --debug
+    datagen -s ./tests/schema.sql -f json -n 10 --record-size 100 --dry-run --debug
+    datagen -s ./tests/schema.json -f avro -n 10 --record-size 100 --dry-run --debug
+    datagen -s ./tests/schema.avsc -f avro -n 10 --record-size 100 --dry-run --debug
+    datagen -s ./tests/schema.sql -f avro -n 10 --record-size 100 --dry-run --debug
 
 ## Cutting a new release
 

README.md

@@ -20,15 +20,16 @@ Usage: datagen [options]
 Fake Data Generator
 
 Options:
-  -V, --version                output the version number
-  -f, --format <char>          The format of the produced data (choices: "json", "avro", default: "json")
-  -s, --schema <char>          Schema file to use
-  -n, --number <char>          Number of records to generate (default: "10", infinite records: "-1")
-  -d, --debug <char>            (choices: "true", "false", default: "false")
-  -w, --wait <int>             Wait time in ms between record production (default: "0")
-  -dr, --dry-run <char>        Dry run (no data will be produced (choices: "true", "false", default: "false")
-  -rs, --record-size <char>    Record size in bytes, eg. 1048576 for 1MB
-  -h, --help                   display help for command
+  -V, --version             output the version number
+  -f, --format <char>       The format of the produced data (choices: "json", "avro", default: "json")
+  -s, --schema <char>       Schema file to use
+  -n, --number <char>       Number of records to generate. For infinite records, use -1 (default: "10")
+  -c, --clean               Clean Kafka topic and schema registry before producing data
+  -dr, --dry-run            Dry run (no data will be produced to Kafka)
+  -d, --debug               Output extra debugging information
+  -w, --wait <int>          Wait time in ms between record production
+  -rs, --record-size <int>  Record size in bytes, eg. 1048576 for 1MB
+  -h, --help                display help for command
 ```
 
 ### Env variables
@@ -146,7 +147,7 @@ docker run \
   --rm -it \
   -v ${PWD}/.env:/app/.env \
   -v ${PWD}/tests/schema.json:/app/blah.json \
-    datagen -s blah.json -n 1 -dr true
+    datagen -s blah.json -n 1 --dry-run
 ```
 
 ### Generate records with sequence numbers
@@ -170,5 +171,5 @@ This is particularly useful when you want to generate a small set of records wit
 Example:
 
 ```
-datagen -s tests/iterationIndex.json -dr true -f json -n 1000
+datagen -s tests/iterationIndex.json --dry-run -f json -n 1000
 ```

datagen.js

@@ -17,7 +17,7 @@ const dataGenerator = require('./src/dataGenerator');
 const fs = require('fs');
 const { program, Option } = require('commander');
 
-program.name('datagen').description('Fake Data Generator').version('0.1.1');
+program.name('datagen').description('Fake Data Generator').version('0.1.2');
 
 program
     .addOption(
@@ -32,21 +32,9 @@ program
             'Number of records to generate. For infinite records, use -1'
         ).default('10')
     )
-    .addOption(
-        new Option('-d, --debug <char>')
-            .choices(['true', 'false'])
-            .default('false')
-    )
-    .addOption(
-        new Option('-c, --clean <char>')
-            .choices(['true', 'false'])
-            .default('false')
-    )
-    .addOption(
-        new Option('-dr, --dry-run <char>', 'Dry run (no data will be produced')
-            .choices(['true', 'false'])
-            .default('false')
-    )
+    .option('-c, --clean', 'Clean Kafka topic and schema registry before producing data')
+    .option('-dr, --dry-run', 'Dry run (no data will be produced to Kafka)')
+    .option('-d, --debug', 'Output extra debugging information')
     .option('-w, --wait <int>', 'Wait time in ms between record production', parseInt)
     .option('-rs, --record-size <int>', 'Record size in bytes, eg. 1048576 for 1MB', parseInt);
 
@@ -69,7 +57,7 @@ global.wait = options.wait;
 global.clean = options.clean;
 global.dryRun = options.dryRun;
 
-if (debug === 'true') {
+if (debug) {
     console.log(options);
 }
 
@@ -114,7 +102,7 @@ if (!wait) {
         process.exit();
     }
 
-    if (clean == 'true') {
+    if (clean) {
         let topics = []
         for (table of parsedSchema){
             topics.push(table._meta.topic)

package.json

@@ -1,7 +1,7 @@
 {
     "name": "@materializeinc/datagen",
     "description": "Materialize Datagen CLI tool",
-    "version": "0.1.1",
+    "version": "0.1.2",
     "license": "Apache-2.0",
     "bin": {
         "@MaterializeInc/datagen": "datagen.js",

src/dataGenerator.js

@@ -27,7 +27,7 @@ async function* asyncGenerator(number) {
 }
 
 function sleep(s) {
-    if (debug === 'true' && wait > 0) {
+    if (debug && wait > 0) {
         alert({
             type: `success`,
             name: `Sleeping for ${s} milliseconds...`,
@@ -38,7 +38,7 @@ function sleep(s) {
 }
 
 async function prepareTopic(topic, dryRun) {
-    if (dryRun == 'true') {
+    if (dryRun) {
         alert({
             type: `success`,
             name: `Dry run: Skipping topic creation...`,
@@ -112,7 +112,7 @@ module.exports = async ({
             }
             for (const topic in megaRecord) {
                 await prepareTopic(topic, dryRun);
-                if (format == 'avro' && dryRun != 'true') {
+                if (format == 'avro' && dryRun !== true) {
                     avroSchemas = await prepareSchema(
                         megaRecord,
                         topic,
@@ -135,7 +135,7 @@ module.exports = async ({
                     record.recordSizePayload = payload;
                 }
 
-                if (dryRun == 'true') {
+                if (dryRun) {
                     alert({
                         type: `success`,
                         name: `Dry run: Skipping record production...`,

src/kafka/cleanKafka.js

@@ -32,7 +32,7 @@ async function deleteSchemaSubjects(topics) {
 
 module.exports = async (format, topics) => {
 
-    if (dryRun == 'true') {
+    if (dryRun) {
         console.log("This is a dry run, so no resources will be deleted")
         return
     }
@@ -55,4 +55,4 @@ module.exports = async (format, topics) => {
         await deleteSchemaSubjects(topics);
     }
 
-};
+};

src/kafka/createTopic.js

@@ -5,7 +5,7 @@ const dotenv = require('dotenv');
 module.exports = async (topic = 'datagen_test_topic') => {
     const kafka = kafkaConfig();
 
-    if (debug === 'true') {
+    if (debug) {
         console.log(`Trying to create topic: ${topic}`);
     }
     // Check if the topic exists in the Kafka cluster if not create it
@@ -43,13 +43,13 @@ async function getReplicationFactor(admin) {
     let replicationFactor = 1;
 
     try {
-        if (debug === 'true') {
+        if (debug) {
             console.log(`Trying to get brokers list...`);
         }
         const brokersList = await admin.describeCluster();
         const brokerId = brokersList.brokers[0].nodeId.toString();
 
-        if (debug === 'true') {
+        if (debug) {
             console.log(`Trying to get default replication factor...`);
         }
 
@@ -58,7 +58,7 @@ async function getReplicationFactor(admin) {
                 resources: [{ type: ConfigResourceTypes.BROKER, name: brokerId, configNames: ['default.replication.factor'] }]
             })
             .catch(err => {
-                if (debug === 'true') {
+                if (debug) {
                     console.log(err);
                 }
             });
@@ -68,7 +68,7 @@ async function getReplicationFactor(admin) {
         ).configValue;
     } catch (err) {
         console.log(`Error getting default replication factor, using 1`);
-        if (debug === 'true') {
+        if (debug) {
             console.log(err);
         }
     }

src/schemas/parseAvroSchema.js

@@ -10,7 +10,7 @@ async function parseAvroSchema(schemaFile) {
         msg: ``
     });
 
-    if (debug === 'true') {
+    if (debug) {
         const parsed = avro.parse(schemaFile);
         console.log(parsed);
     }

src/schemas/parseJsonSchema.js

@@ -7,7 +7,7 @@ async function parseJsonSchema(schemaFile) {
         msg: ``
     });
 
-    if (debug === 'true') {
+    if (debug) {
         const parsed = JSON.parse(schemaFile);
         console.log(parsed);
     }

src/schemas/parseSqlSchema.js

@@ -34,7 +34,7 @@ async function parseSqlSchema(schemaFile) {
     // Convert the schema to JSON
     tables = await convertSqlSchemaToJson(tables);
 
-    if (debug === 'true') {
+    if (debug) {
         console.log(tables, null, 3);
     }
 

tests/datagen.test.js

@@ -2,7 +2,7 @@ const { Command } = require('commander');
 const { execSync } = require('child_process');
 
 const datagen = args => {
-    return execSync(`node ./datagen.js -dr true ${args}`).toString();
+    return execSync(`node ./datagen.js --dry-run ${args}`).toString();
 };
 
 describe('Test datagen help', () => {