#239 avro extractor union record (#271)

sngular · Jul 4, 2022 · 60d3700 · 60d3700
1 parent 1d50ab2
commit 60d3700
Show file tree

Hide file tree

Showing 8 changed files with 225 additions and 144 deletions.
diff --git a/pom-maven-central.xml b/pom-maven-central.xml
@@ -7,7 +7,7 @@
 
   <artifactId>kloadgen</artifactId>
 
-  <version>4.10.1</version>
+  <version>4.11.0</version>
   <name>KLoadGen</name>
   <description>Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial data base on Data specification</description>
   <url>https://corunet.github.io/kloadgen/</url>

diff --git a/pom.xml b/pom.xml
@@ -7,7 +7,7 @@
 
   <artifactId>kloadgen</artifactId>
 
-  <version>4.10.1</version>
+  <version>4.11.0</version>
   <name>KLoadGen</name>
   <description>Load Generation Jmeter plugin for Kafka Cluster. Supporting AVRO, JSON Schema and Protobuf schema types. Generate Artificial data base on Data specification</description>
   <url>https://corunet.github.io/kloadgen/</url>

diff --git a/src/main/java/net/coru/kloadgen/extractor/extractors/AvroExtractor.java b/src/main/java/net/coru/kloadgen/extractor/extractors/AvroExtractor.java
@@ -87,18 +87,14 @@ private void processUnionField(final String fieldName, final List<FieldValueMapp
 
   private void processRecordFieldList(final String fieldName, final String splitter, final List<FieldValueMapping> internalFields,
       final List<FieldValueMapping> completeFieldList) {
-    if (completeFieldList.isEmpty()) {
-      internalFields.forEach(internalField -> {
-        if (internalField.getFieldName().startsWith(fieldName + ".")) {
-          internalField.setFieldName(fieldName + internalField.getFieldName().replace(fieldName, splitter.replace(".", "")));
-        } else {
-          internalField.setFieldName(fieldName + splitter + internalField.getFieldName());
-        }
-        completeFieldList.add(internalField);
-      });
-    } else {
-      internalFields.forEach(internalField -> internalField.setFieldName(fieldName + splitter + internalField.getFieldName()));
-    }
+    internalFields.forEach(internalField -> {
+      if (internalField.getFieldName().startsWith(fieldName + ".")) {
+        internalField.setFieldName(fieldName + internalField.getFieldName().replace(fieldName, splitter.replace(".", "")));
+      } else {
+        internalField.setFieldName(fieldName + splitter + internalField.getFieldName());
+      }
+      completeFieldList.add(internalField);
+    });
   }
 
   private List<FieldValueMapping> processFieldList(final List<Schema.Field> fieldList, final boolean isAncestorRequired) {

diff --git a/src/main/java/net/coru/kloadgen/processor/AvroSchemaProcessor.java b/src/main/java/net/coru/kloadgen/processor/AvroSchemaProcessor.java
diff --git a/src/main/java/net/coru/kloadgen/processor/ProtobufSchemaProcessor.java b/src/main/java/net/coru/kloadgen/processor/ProtobufSchemaProcessor.java
diff --git a/src/main/java/net/coru/kloadgen/processor/SchemaProcessorLib.java b/src/main/java/net/coru/kloadgen/processor/SchemaProcessorLib.java
@@ -6,15 +6,6 @@
 
 package net.coru.kloadgen.processor;
 
-import java.util.ArrayDeque;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
 import net.coru.kloadgen.model.FieldValueMapping;
 import net.coru.kloadgen.randomtool.random.RandomArray;
 import net.coru.kloadgen.randomtool.random.RandomMap;
@@ -24,6 +15,10 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.jmeter.threads.JMeterContextService;
 
+import java.util.*;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
 public final class SchemaProcessorLib {
 
   private static final Map<String, Object> CONTEXT = new HashMap<>();
@@ -123,31 +118,24 @@ static Integer calculateMapSize(final String fieldName, final String methodName)
 
   }
 
-  static String cleanUpPath(final FieldValueMapping fieldValueMapping, final String fieldName) {
-    int startPosition = 0;
-    String cleanPath;
-    if (StringUtils.isNotEmpty(fieldName)) {
-      startPosition = fieldValueMapping.getFieldName().indexOf(fieldName) + fieldName.length() + 1;
-    }
-    cleanPath = fieldValueMapping.getFieldName().substring(startPosition);
-    if (cleanPath.matches("^(\\d*:*]).*$")) {
-      cleanPath = cleanPath.substring(cleanPath.indexOf(".") + 1);
-    }
-    return cleanPath;
+  static String cleanUpPath(final FieldValueMapping fieldValueMapping, final int level) {
+    final String[] splitPath = fieldValueMapping.getFieldName().split("\\.");
+    final List<String> splitPathAux = Arrays.asList(Arrays.copyOfRange(splitPath, level, splitPath.length));
+    return String.join(".",splitPathAux);
   }
 
-  static String getCleanMethodName(final FieldValueMapping fieldValueMapping, final String fieldName) {
-    return getFullMethodName(fieldValueMapping, fieldName).replaceAll("\\[[0-9]*:?]", "");
+  static String getCleanMethodName(final FieldValueMapping fieldValueMapping, final int level) {
+    return getFullMethodName(fieldValueMapping, level).replaceAll("\\[[0-9]*:?]", "");
   }
 
-  static String getFullMethodName(final FieldValueMapping fieldValueMapping, final String fieldName) {
-    final String pathToClean = cleanUpPath(fieldValueMapping, fieldName);
+  static String getFullMethodName(final FieldValueMapping fieldValueMapping, final int level) {
+    final String pathToClean = cleanUpPath(fieldValueMapping, level);
     final int endOfField = pathToClean.contains(".") ? pathToClean.indexOf(".") : pathToClean.length();
     return pathToClean.substring(0, endOfField);
   }
 
-  static String getMapCleanMethodName(final FieldValueMapping fieldValueMapping, final String fieldName) {
-    final String pathToClean = cleanUpPath(fieldValueMapping, fieldName);
+  static String getMapCleanMethodName(final FieldValueMapping fieldValueMapping, final int level) {
+    final String pathToClean = cleanUpPath(fieldValueMapping, level);
     final int endOfField = pathToClean.contains("[") ? pathToClean.indexOf("[") : 0;
     return pathToClean.substring(0, endOfField).replaceAll("\\[\\d*:?]", "");
   }

diff --git a/src/test/java/net/coru/kloadgen/extractor/extractors/AvroExtractorTest.java b/src/test/java/net/coru/kloadgen/extractor/extractors/AvroExtractorTest.java
@@ -130,4 +130,28 @@ void testFlatPropertiesOptionalArray() throws Exception {
         );
   }
 
+  @Test
+  @DisplayName("Should extract Union Record")
+  void testFlatPropertiesUnionRecordAvros() throws Exception {
+
+    String testFile = fileHelper.getContent("/avro-files/testUnionRecord.avsc");
+    ParsedSchema parsedSchema = avroExtractor.getParsedSchema(testFile);
+    Schema schema = (Schema) parsedSchema.rawSchema();
+
+    List<FieldValueMapping> fieldValueMappingList = avroExtractor.processSchema(schema);
+
+    assertThat(fieldValueMappingList)
+      .hasSize(8)
+      .containsExactlyInAnyOrder(
+        FieldValueMapping.builder().fieldName("validateInnerObject.attribute1").fieldType("string").fieldValueList("").valueLength(0).required(true).isAncestorRequired(true).build(),
+        FieldValueMapping.builder().fieldName("validateInnerObject.attribute2").fieldType("string").fieldValueList("").valueLength(0).required(true).isAncestorRequired(true).build(),
+        FieldValueMapping.builder().fieldName("products[].Price.price").fieldType("string").fieldValueList("").valueLength(0).required(true).isAncestorRequired(true).build(),
+        FieldValueMapping.builder().fieldName("products[].Price.priceType").fieldType("string").fieldValueList("").valueLength(0).required(true).isAncestorRequired(true).build(),
+        FieldValueMapping.builder().fieldName("products[].Price.currency").fieldType("string").fieldValueList("").valueLength(0).required(true).isAncestorRequired(true).build(),
+        FieldValueMapping.builder().fieldName("products[].Price.discount").fieldType("string").fieldValueList("").valueLength(0).required(true).isAncestorRequired(true).build(),
+        FieldValueMapping.builder().fieldName("products[].Price.validateInnerObject.attribute1").fieldType("string").fieldValueList("").valueLength(0).required(true).isAncestorRequired(true).build(),
+        FieldValueMapping.builder().fieldName("products[].Price.validateInnerObject.attribute2").fieldType("string").fieldValueList("").valueLength(0).required(true).isAncestorRequired(true).build()
+      );
+  }
+
 }
diff --git a/src/test/resources/avro-files/testUnionRecord.avsc b/src/test/resources/avro-files/testUnionRecord.avsc
@@ -0,0 +1,86 @@
+{
+   "fields":[
+      {
+         "name":"validateInnerObject",
+         "type":{
+            "fields":[
+               {
+                  "name":"attribute1",
+                  "type":{
+                     "avro.java.string":"String",
+                     "type":"string"
+                  }
+               },
+               {
+                  "name":"attribute2",
+                  "type":{
+                     "avro.java.string":"String",
+                     "type":"string"
+                  }
+               }
+            ],
+            "name":"ValidateInnerObject",
+            "type":"record"
+         }
+      },
+      {
+         "name":"products",
+         "type":{
+            "items":{
+               "fields":[
+                  {
+                     "name":"Price",
+                     "type":{
+                        "fields":[
+                           {
+                              "name":"price",
+                              "type":{
+                                 "avro.java.string":"String",
+                                 "type":"string"
+                              }
+                           },
+                           {
+                              "name":"priceType",
+                              "type":{
+                                 "avro.java.string":"String",
+                                 "type":"string"
+                              }
+                           },
+                           {
+                              "name":"currency",
+                              "type":{
+                                 "avro.java.string":"String",
+                                 "type":"string"
+                              }
+                           },
+                           {
+                              "name":"discount",
+                              "type":{
+                                 "avro.java.string":"String",
+                                 "type":"string"
+                              }
+                           },
+                           {
+                              "name":"validateInnerObject",
+                              "type":[
+                                 "null",
+                                 "ValidateInnerObject"
+                              ]
+                           }
+                        ],
+                        "name":"InnerPrice",
+                        "type":"record"
+                     }
+                  }
+               ],
+               "name":"Product",
+               "type":"record"
+            },
+            "type":"array"
+         }
+      }
+   ],
+   "name":"ProductInnerObject",
+   "namespace":"net.coru",
+   "type":"record"
+}