Feature/cdutz/go ads ng (Streamlining of PLC4X API in PLC4Go and PLC4J)

Jenkinsfile

@@ -38,6 +38,8 @@ pipeline {
         // Test failures will be handled by the jenkins junit steps and mark the build as unstable.
         MVN_TEST_FAIL_IGNORE = '-Dmaven.test.failure.ignore=true'
 
+        ENABLE_ALL_TESTS = true
+
         SONARCLOUD_PARAMS = "-Dsonar.host.url=https://sonarcloud.io -Dsonar.organization=apache -Dsonar.projectKey=apache_plc4x -Dsonar.branch.name=develop"
     }
 

code-generation/language-go/src/main/java/org/apache/plc4x/language/go/GoLanguageTemplateHelper.java

@@ -20,6 +20,7 @@
 
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.math.NumberUtils;
+import org.apache.commons.text.CaseUtils;
 import org.apache.plc4x.plugins.codegenerator.language.mspec.model.definitions.DefaultArgument;
 import org.apache.plc4x.plugins.codegenerator.language.mspec.model.references.DefaultBooleanTypeReference;
 import org.apache.plc4x.plugins.codegenerator.language.mspec.model.references.DefaultFloatTypeReference;
@@ -59,9 +60,18 @@ public String fileName(String protocolName, String languageName, String language
             String.join("", languageFlavorName.split("\\-"));
     }
 
+    public String getSanitizedPackageName() {
+        String sanitizedName = getProtocolName().replaceAll("-", "");
+        sanitizedName = sanitizedName.replaceAll("\\.", "/");
+        sanitizedName = sanitizedName.toLowerCase();
+        return sanitizedName;
+    }
+
     // TODO: check if protocol name can be enforced to only contain valid chars
     public String getSanitizedProtocolName() {
-        return getProtocolName().replaceAll("-", "");
+        String sanitizedName = getProtocolName().replaceAll("-", "");
+        sanitizedName = CaseUtils.toCamelCase(sanitizedName, false, '.');
+        return sanitizedName;
     }
 
     public String packageName(String languageFlavorName) {
@@ -712,6 +722,10 @@ private String toLiteralTermExpression(Field field, TypeReference fieldType, Ter
             return tracer + "\"" + ((StringLiteral) term).getValue() + "\"";
         } else if (term instanceof VariableLiteral) {
             tracer = tracer.dive("variable literal instanceOf");
+            VariableLiteral variableLiteral = (VariableLiteral) term;
+            if ("curPos".equals(((VariableLiteral) term).getName())) {
+                return "(positionAware.GetPos() - startPos)";
+            }
             return tracer + toVariableExpression(field, fieldType, (VariableLiteral) term, parserArguments, serializerArguments, serialize, suppressPointerAccess);
         } else {
             throw new RuntimeException("Unsupported Literal type " + term.getClass().getName());
@@ -1540,4 +1554,21 @@ public String capitalize(String str) {
         String cleanedString = dummyTracer.removeTraces(str);
         return extractedTrace + StringUtils.capitalize(cleanedString);
     }
+
+    public String getEndiannessOptions(boolean read, boolean separatorPrefix) {
+        Optional<Term> byteOrder = thisType.getAttribute("byteOrder");
+        if (byteOrder.isPresent()) {
+            emitRequiredImport("encoding/binary");
+
+            String functionName = read ? "WithByteOrderForReadBufferByteBased" : "WithByteOrderForByteBasedBuffer";
+            String byteOrderValue = ((VariableLiteral) byteOrder.get()).getName();
+            if("BIG_ENDIAN".equals(byteOrderValue)) {
+                return (separatorPrefix ? ", " : "") + "utils." + functionName + "(binary.BigEndian)";
+            } else if ("LITTLE_ENDIAN".equals(byteOrderValue)) {
+                return (separatorPrefix ? ", " : "") + "utils." + functionName + "(binary.LittleEndian)";
+            }
+        }
+        return "";
+    }
+
 }

code-generation/language-go/src/main/resources/templates/go/complex-type-template.go.ftlh

@@ -747,7 +747,7 @@ func (m *_${type.name}) GetLengthInBytes() uint16 {
 <#assign parserArgumentList><#if hasParserArguments><#list parserArguments as parserArgument>${parserArgument.name} ${helper.getLanguageTypeNameForTypeReference(parserArgument.type)}<#sep>, </#sep></#list></#if></#assign>
 <#assign parserArgumentNameList><#if hasParserArguments><#list parserArguments as parserArgument>${parserArgument.name}<#sep>, </#sep></#list></#if></#assign>
 func ${type.name}Parse(theBytes []byte<#if hasParserArguments>, ${parserArgumentList}</#if>) (${type.name}, error) {
-	return ${type.name}ParseWithBuffer(utils.NewReadBufferByteBased(theBytes, utils.WithByteOrderForReadBufferByteBased(binary.BigEndian))<#if hasParserArguments>, ${parserArgumentNameList}</#if>) <@emitImport import="encoding/binary" /> // TODO: get endianness from mspec
+	return ${type.name}ParseWithBuffer(utils.NewReadBufferByteBased(theBytes${helper.getEndiannessOptions(true, true)})<#if hasParserArguments>, ${parserArgumentNameList}</#if>)
 }
 
 func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer<#if hasParserArguments>, ${parserArgumentList}</#if>) (${type.name}, error) {
@@ -762,9 +762,6 @@ func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer<#if hasParserArgume
 	var startPos = positionAware.GetPos()
 	_ = startPos
 	</#if>
-	<#if helper.requiresCurPos()>
-	var curPos uint16
-	</#if>
 	<#assign reservedFieldIndex=0>
 	<#list type.fields as field>
 		<#switch field.typeName>
@@ -799,10 +796,6 @@ func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer<#if hasParserArgume
 	if pullErr := readBuffer.PullContext("${arrayField.name}", utils.WithRenderAsList(true)); pullErr != nil {
 		return nil, errors.Wrap(pullErr, "Error pulling for ${arrayField.name}")<@emitImport import="github.com/pkg/errors" />
 	}
-			<#-- Only update curPos if the length expression uses it -->
-					<#if arrayField.loopExpression.contains("curPos")>
-	curPos = positionAware.GetPos() - startPos
-					</#if>
 			<#-- If this is a count array, we can directly initialize an array with the given size -->
 					<#if field.isCountArrayField()>
 	// Count array
@@ -897,10 +890,6 @@ func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer<#if hasParserArgume
                         <#else>
 			${arrayField.name} = append(${arrayField.name}, _item.(${arrayElementType.asComplexTypeReference().orElseThrow().name}))
                         </#if>
-						<#-- After parsing, update the current position, but only if it's needed -->
-							<#if arrayField.loopExpression.contains("curPos")>
-			curPos = positionAware.GetPos() - startPos
-							</#if>
 		}
 	}
 					<#-- A terminated array keeps on reading data as long as the termination expression evaluates to false -->
@@ -943,11 +932,6 @@ func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer<#if hasParserArgume
 						<#else>
 			${arrayField.name} = append(${arrayField.name}, _item.(${arrayElementType.asComplexTypeReference().orElseThrow().name}))
 						</#if>
-
-						<#-- After parsing, update the current position, but only if it's needed -->
-							<#if arrayField.loopExpression.contains("curPos")>
-			curPos = positionAware.GetPos() - startPos
-							</#if>
 		}
 	}
 					</#if>
@@ -1066,10 +1050,6 @@ func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer<#if hasParserArgume
 		return nil, errors.Wrap(pullErr, "Error pulling for ${manualArrayField.name}")<@emitImport import="github.com/pkg/errors" />
 	}
 	// Manual Array Field (${manualArrayField.name})
-			<#-- Only update curPos if the length expression uses it -->
-				<#if manualArrayField.loopExpression.contains("curPos")>
-	curPos = positionAware.GetPos() - startPos
-				</#if>
 			<#-- If this is a count array, we can directly initialize an array with the given size -->
 				<#if field.isCountArrayField()>
 	// Count array
@@ -1092,10 +1072,6 @@ func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer<#if hasParserArgume
 	${manualArrayField.name}EndPos := positionAware.GetPos() + _${manualArrayField.name}Length
 	for ;positionAware.GetPos() < ${manualArrayField.name}EndPos; {
 		_${manualArrayField.name}List = append(_${manualArrayField.name}List, ((${helper.getLanguageTypeNameForField(field)}) (${helper.toParseExpression(manualArrayField, arrayElementType, manualArrayField.parseExpression, parserArguments)})))
-					<#-- After parsing, update the current position, but only if it's needed -->
-						<#if manualArrayField.loopExpression.contains("curPos")>
-		curPos = positionAware.GetPos() - startPos
-						</#if>
 	}
 					<#-- A terminated array keeps on reading data as long as the termination expression evaluates to false -->
 					<#elseif field.isTerminatedArrayField()>
@@ -1107,10 +1083,6 @@ func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer<#if hasParserArgume
 		for ;!((bool) (${helper.toParseExpression(manualArrayField, helper.boolTypeReference, manualArrayField.loopExpression, parserArguments, true)})); {
 		_${manualArrayField.name}List = append(_${manualArrayField.name}List, ((${helper.getLanguageTypeNameForTypeReference(arrayElementType)}) (${helper.toParseExpression(manualArrayField, arrayElementType, manualArrayField.parseExpression, parserArguments)})))
 
-					<#-- After parsing, update the current position, but only if it's needed -->
-						<#if manualArrayField.loopExpression.contains("curPos")>
-			curPos = positionAware.GetPos() - startPos
-						</#if>
 		}
 	}
 					</#if>
@@ -1153,9 +1125,6 @@ func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer<#if hasParserArgume
 				<#assign optionalField = field.asOptionalField().orElseThrow()>
 
 	// Optional Field (${optionalField.name}) (Can be skipped, if a given expression evaluates to false)
-				<#if optionalField.conditionExpression.present && optionalField.conditionExpression.get().contains("curPos")>
-	curPos = positionAware.GetPos() - startPos
-				</#if>
 	var ${optionalField.name} <#if !optionalField.type.isComplexTypeReference()>*</#if>${helper.getLanguageTypeNameForField(field)} = nil
 	<#if optionalField.conditionExpression.present>
 	if ${helper.toBooleanParseExpression(optionalField, optionalField.conditionExpression.get(), parserArguments)} </#if>{
@@ -1223,9 +1192,6 @@ func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer<#if hasParserArgume
 				<#assign assertField = field.asAssertField().orElseThrow()>
 
 	// Assert Field (${assertField.name}) (Can be skipped, if a given expression evaluates to false)
-				<#if assertField.conditionExpression.contains("curPos")>
-	curPos = positionAware.GetPos() - startPos
-				</#if>
 			<#if assertField.type.isSimpleTypeReference()>
 	${assertField.name}, _err := ${helper.getReadBufferReadMethodCall(assertField.name, assertField.type.asSimpleTypeReference().orElseThrow(), assertField)}
 	if _err != nil {
@@ -1571,7 +1537,7 @@ func (pm *_${type.name}) SerializeParent(writeBuffer utils.WriteBuffer, child ${
 	_ = m
 <#else>
 func (m *_${type.name}) Serialize() ([]byte, error) {
-	wb := utils.NewWriteBufferByteBased(utils.WithByteOrderForByteBasedBuffer(binary.BigEndian), utils.WithInitialSizeForByteBasedBuffer(int(m.GetLengthInBytes())))<@emitImport import="encoding/binary" /> // TODO: get endianness from mspec
+	wb := utils.NewWriteBufferByteBased(utils.WithInitialSizeForByteBasedBuffer(int(m.GetLengthInBytes()))${helper.getEndiannessOptions(false, true)})
 	if err := m.SerializeWithWriteBuffer(wb); err != nil {
 		return nil, err
 	}

code-generation/language-go/src/main/resources/templates/go/data-io-template.go.ftlh

@@ -69,7 +69,7 @@ import (
 <#-- TODO: the code below implies that parserArguments will be null if not present... not pretty  -->
 <#if type.parserArguments.isPresent()><#assign parserArguments=type.parserArguments.orElseThrow()></#if>
 func ${type.name}Parse(theBytes []byte<#if parserArguments?has_content>, <#list parserArguments as parserArgument>${parserArgument.name} <#if parserArgument.type.isNonSimpleTypeReference() && !parserArgument.type.isEnumTypeReference()>I</#if>${helper.getLanguageTypeNameForTypeReference(parserArgument.type)}<#sep>, </#sep></#list></#if>) (api.PlcValue, error) {
-	return ${type.name}ParseWithBuffer(utils.NewReadBufferByteBased(theBytes, utils.WithByteOrderForReadBufferByteBased(binary.BigEndian))<#if parserArguments?has_content>, <#list parserArguments as parserArgument>${parserArgument.name}<#sep>, </#sep></#list></#if>) // TODO: get endianness from mspec
+	return ${type.name}ParseWithBuffer(utils.NewReadBufferByteBased(theBytes${helper.getEndiannessOptions(true, true)})<#if parserArguments?has_content>, <#list parserArguments as parserArgument>${parserArgument.name}<#sep>, </#sep></#list></#if>)
 }
 
 func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer<#if parserArguments?has_content>, <#list parserArguments as parserArgument>${parserArgument.name} <#if parserArgument.type.isNonSimpleTypeReference() && !parserArgument.type.isEnumTypeReference()>I</#if>${helper.getLanguageTypeNameForTypeReference(parserArgument.type)}<#sep>, </#sep></#list></#if>) (api.PlcValue, error) {
@@ -209,7 +209,7 @@ func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer<#if parserArguments
 }
 
 func ${type.name}Serialize(value api.PlcValue<#if parserArguments?has_content>, <#list parserArguments as parserArgument>${parserArgument.name} <#if parserArgument.type.isNonSimpleTypeReference() && !parserArgument.type.isEnumTypeReference()>I</#if>${helper.getLanguageTypeNameForTypeReference(parserArgument.type)}<#sep>, </#sep></#list></#if>) ([]byte, error) {
-	wb := utils.NewWriteBufferByteBased(utils.WithByteOrderForByteBasedBuffer(binary.BigEndian))<@emitImport import="encoding/binary" /> // TODO: get endianness from mspec
+	wb := utils.NewWriteBufferByteBased(${helper.getEndiannessOptions(false, false)})
 	if err := ${type.name}SerializeWithWriteBuffer(wb, value<#if parserArguments?has_content>, <#list parserArguments as parserArgument>${parserArgument.name}<#sep>, </#sep></#list></#if>); err != nil {
 		return nil, err
 	}

code-generation/language-go/src/main/resources/templates/go/enum-template.go.ftlh

@@ -49,8 +49,6 @@ ${helper.fileName(protocolName, languageName, outputFlavor)?replace(".", "/")}/m
 package model
 
 import (
-	"encoding/binary"
-
 	"github.com/apache/plc4x/plc4go/spi/utils"
 	"github.com/pkg/errors"
 )
@@ -180,7 +178,7 @@ func (m ${type.name}) GetLengthInBytes() uint16 {
 	<#assign simpleTypeReference = type.type.orElseThrow().asSimpleTypeReference().orElseThrow()>
 	<#if simpleTypeReference.getSizeInBits() != -1>
 func ${type.name}Parse(theBytes []byte) (${type.name}, error) {
-	return ${type.name}ParseWithBuffer(utils.NewReadBufferByteBased(theBytes, utils.WithByteOrderForReadBufferByteBased(binary.BigEndian))) // TODO: get endianness from mspec
+	return ${type.name}ParseWithBuffer(utils.NewReadBufferByteBased(theBytes${helper.getEndiannessOptions(true, true)}))
 }
 
 func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer) (${type.name}, error) {
@@ -197,7 +195,7 @@ func ${type.name}ParseWithBuffer(readBuffer utils.ReadBuffer) (${type.name}, err
 }
 
 func (e ${type.name}) Serialize() ([]byte, error) {
-	wb := utils.NewWriteBufferByteBased(utils.WithByteOrderForByteBasedBuffer(binary.BigEndian)) // TODO: get endianness from mspec
+	wb := utils.NewWriteBufferByteBased(${helper.getEndiannessOptions(false, false)})
 	if err := e.SerializeWithWriteBuffer(wb); err != nil {
 		return nil, err
 	}

code-generation/language-go/src/main/resources/templates/go/parser-factory-template.go.ftlh

@@ -48,7 +48,7 @@ ${helper.fileName(protocolName, languageName, outputFlavor)?replace(".", "/")}/P
 package ${outputFlavor?replace("-","")}
 
 import (
-	"github.com/apache/plc4x/plc4go/protocols/${helper.getSanitizedProtocolName()}/${outputFlavor?replace("-","")}/model"
+	"github.com/apache/plc4x/plc4go/protocols/${helper.getSanitizedPackageName()}/${outputFlavor?replace("-","")}/model"
 	"github.com/apache/plc4x/plc4go/spi/utils"
 	"github.com/pkg/errors"
 )

code-generation/language-go/src/main/resources/templates/go/xml-parser-factory-template.go.ftlh

@@ -48,7 +48,7 @@ ${helper.fileName(protocolName, languageName, outputFlavor)?replace(".", "/")}/X
 package ${outputFlavor?replace("-","")}
 
 import (
-	"github.com/apache/plc4x/plc4go/protocols/${helper.getSanitizedProtocolName()}/${outputFlavor?replace("-","")}/model"
+	"github.com/apache/plc4x/plc4go/protocols/${helper.getSanitizedPackageName()}/${outputFlavor?replace("-","")}/model"
 	"github.com/apache/plc4x/plc4go/spi/utils"
 	"github.com/pkg/errors"
     "strings"

code-generation/language-java/src/main/resources/templates/java/data-io-template.java.ftlh

@@ -48,7 +48,7 @@ package ${helper.packageName(protocolName, languageName, outputFlavor)};
 
 import static org.apache.plc4x.java.spi.generation.StaticHelper.*;
 
-import org.apache.plc4x.java.api.model.PlcField;
+import org.apache.plc4x.java.api.model.PlcTag;
 import org.apache.plc4x.java.api.value.*;
 import org.apache.plc4x.java.spi.generation.EvaluationHelper;
 import org.apache.plc4x.java.spi.generation.ParseException;

code-generation/protocol-base-mspec/src/main/antlr4/org/apache/plc4x/plugins/codegenerator/language/mspec/MSpec.g4

@@ -343,13 +343,13 @@ STRING_CHARACTER
 // Stuff we just want to ignore
 
 LINE_COMMENT
- : '//' ~[\r\n]* -> skip
+ : '//' ~[\r\n]* -> channel(HIDDEN)
  ;
 
 BLOCK_COMMENT
- : '/*' .*? '*/' -> skip
+ : '/*' .*? '*/' -> channel(HIDDEN)
  ;
 
 WS
- : [ \t\r\n\u000C]+ -> skip
+ : [ \t\r\n\u000C]+ -> channel(HIDDEN)
  ;

plc4c/api/include/plc4c/read.h

@@ -31,13 +31,13 @@ extern "C" {
 void plc4c_read_request_destroy(plc4c_read_request *read_request);
 
 /**
- * Function for adding a new item to a given request.
+ * Function for adding a new tag, based on a given address to a given request.
  * @param read_request the read-request.
  * @param address address string
  * @return return code
  */
-plc4c_return_code plc4c_read_request_add_item(plc4c_read_request *read_request,
-                                              char *field_name,
+plc4c_return_code plc4c_read_request_add_tag_address(plc4c_read_request *read_request,
+                                              char *tag_name,
                                               char *address);
 
 /**