Model optional fields in property-based tests.

Previously, we were only supplying required fields in the schema. Now,
we allow fields and primitive types to be marked as optional. Note that
they can be marked as optional in two places.

Optional => null value.

There are some oddities around specifying `nullValue`s:
#429

We currently use the default `nullValue`s for types rather than
generating arbitrary ones.

sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/arbitraries/SbeArbitraries.java

@@ -112,6 +112,7 @@ private static Arbitrary<EncodedDataTypeSchema> encodedDataTypeSchema()
         return Combinators.combine(
             Arbitraries.of(PrimitiveType.values()),
             Arbitraries.of(1, 1, 1, 2, 13),
+            presence(),
             Arbitraries.of(true, false)
         ).as(EncodedDataTypeSchema::new);
     }
@@ -190,12 +191,26 @@ private static Arbitrary<GroupSchema> groupSchema(final int depth)
             groupSchema(depth - 1).list().ofMaxSize(3);
 
         return Combinators.combine(
-            withDuplicates(2, typeSchema(MAX_COMPOSITE_DEPTH).list().ofMaxSize(5)),
+            withDuplicates(
+                2,
+                Combinators.combine(
+                    typeSchema(MAX_COMPOSITE_DEPTH),
+                    presence()
+                ).as(FieldSchema::new).list().ofMaxSize(5)
+            ),
             subGroups,
             varDataSchema().list().ofMaxSize(3)
         ).as(GroupSchema::new);
     }
 
+    private static Arbitrary<Encoding.Presence> presence()
+    {
+        return Arbitraries.of(
+            Encoding.Presence.REQUIRED,
+            Encoding.Presence.OPTIONAL
+        );
+    }
+
     private static Arbitrary<VarDataSchema> varDataSchema()
     {
         return Arbitraries.of(VarDataSchema.Encoding.values())
@@ -205,7 +220,14 @@ private static Arbitrary<VarDataSchema> varDataSchema()
     public static Arbitrary<MessageSchema> messageSchema()
     {
         return Combinators.combine(
-            withDuplicates(3, typeSchema(MAX_COMPOSITE_DEPTH).list().ofMaxSize(10)),
+
+            withDuplicates(
+                3,
+                Combinators.combine(
+                    typeSchema(MAX_COMPOSITE_DEPTH),
+                    presence()
+                ).as(FieldSchema::new).list().ofMaxSize(10)
+            ),
             groupSchema(MAX_GROUP_DEPTH).list().ofMaxSize(3),
             varDataSchema().list().ofMaxSize(3)
         ).as(MessageSchema::new);
@@ -752,7 +774,12 @@ public static Arbitrary<EncodedMessage> encodedMessage(final CharGenerationMode
             final String xml = TestXmlSchemaWriter.writeString(testSchema);
             try (InputStream in = new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8)))
             {
-                final uk.co.real_logic.sbe.xml.MessageSchema parsedSchema = parse(in, ParserOptions.DEFAULT);
+                final ParserOptions options = ParserOptions.builder()
+                    .suppressOutput(false)
+                    .warningsFatal(true)
+                    .stopOnError(true)
+                    .build();
+                final uk.co.real_logic.sbe.xml.MessageSchema parsedSchema = parse(in, options);
                 final Ir ir = new IrGenerator().generate(parsedSchema);
                 return SbeArbitraries.messageValueEncoder(ir, testSchema.templateId(), mode)
                     .map(encoder ->

sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/EncodedDataTypeSchema.java

@@ -17,21 +17,25 @@
 package uk.co.real_logic.sbe.properties.schema;
 
 import uk.co.real_logic.sbe.PrimitiveType;
+import uk.co.real_logic.sbe.ir.Encoding;
 
 public final class EncodedDataTypeSchema implements TypeSchema
 {
     private final PrimitiveType primitiveType;
     private final int length;
+    private final Encoding.Presence presence;
     private final boolean isEmbedded;
 
     public EncodedDataTypeSchema(
         final PrimitiveType primitiveType,
         final int length,
+        final Encoding.Presence presence,
         final boolean isEmbedded
     )
     {
         this.primitiveType = primitiveType;
         this.length = length;
+        this.presence = presence;
         this.isEmbedded = isEmbedded;
     }
 
@@ -45,6 +49,11 @@ public int length()
         return length;
     }
 
+    public Encoding.Presence presence()
+    {
+        return presence;
+    }
+
     @Override
     public boolean isEmbedded()
     {

sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/FieldSchema.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2013-2023 Real Logic Limited.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package uk.co.real_logic.sbe.properties.schema;
+
+import uk.co.real_logic.sbe.ir.Encoding;
+
+public final class FieldSchema
+{
+    private final TypeSchema type;
+    private final Encoding.Presence presence;
+
+    public FieldSchema(
+        final TypeSchema type,
+        final Encoding.Presence presence
+    )
+    {
+        this.type = type;
+        this.presence = presence;
+    }
+
+    public TypeSchema type()
+    {
+        return type;
+    }
+
+    public Encoding.Presence presence()
+    {
+        return presence;
+    }
+}

sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/GroupSchema.java

@@ -20,12 +20,12 @@
 
 public final class GroupSchema
 {
-    private final List<TypeSchema> blockFields;
+    private final List<FieldSchema> blockFields;
     private final List<GroupSchema> groups;
     private final List<VarDataSchema> varData;
 
     public GroupSchema(
-        final List<TypeSchema> blockFields,
+        final List<FieldSchema> blockFields,
         final List<GroupSchema> groups,
         final List<VarDataSchema> varData)
     {
@@ -34,7 +34,7 @@ public GroupSchema(
         this.varData = varData;
     }
 
-    public List<TypeSchema> blockFields()
+    public List<FieldSchema> blockFields()
     {
         return blockFields;
     }

sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/MessageSchema.java

@@ -20,12 +20,12 @@
 
 public final class MessageSchema
 {
-    private final List<TypeSchema> blockFields;
+    private final List<FieldSchema> blockFields;
     private final List<GroupSchema> groups;
     private final List<VarDataSchema> varData;
 
     public MessageSchema(
-        final List<TypeSchema> blockFields,
+        final List<FieldSchema> blockFields,
         final List<GroupSchema> groups,
         final List<VarDataSchema> varData
     )
@@ -45,7 +45,7 @@ public short templateId()
         return 1;
     }
 
-    public List<TypeSchema> blockFields()
+    public List<FieldSchema> blockFields()
     {
         return blockFields;
     }

sbe-tool/src/propertyTest/java/uk/co/real_logic/sbe/properties/schema/TestXmlSchemaWriter.java

@@ -16,6 +16,7 @@
 
 package uk.co.real_logic.sbe.properties.schema;
 
+import uk.co.real_logic.sbe.ir.Encoding;
 import org.agrona.collections.MutableInteger;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
@@ -24,6 +25,7 @@
 import java.io.StringWriter;
 import java.util.*;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
 import javax.xml.transform.OutputKeys;
@@ -85,7 +87,7 @@ private static void writeTo(
                 visitedTypes,
                 topLevelTypes,
                 typeSchemaConverter,
-                schema.blockFields(),
+                schema.blockFields().stream().map(FieldSchema::type).collect(Collectors.toList()),
                 schema.groups());
 
             final Element message = document.createElement("sbe:message");
@@ -128,25 +130,26 @@ private static void writeTo(
     private static void appendMembers(
         final Document document,
         final HashMap<TypeSchema, String> typeToName,
-        final List<TypeSchema> blockFields,
+        final List<FieldSchema> blockFields,
         final List<GroupSchema> groups,
         final List<VarDataSchema> varData,
         final MutableInteger nextMemberId,
         final Element parent)
     {
-        for (final TypeSchema field : blockFields)
+        for (final FieldSchema field : blockFields)
         {
             final int id = nextMemberId.getAndIncrement();
 
-            final boolean usePrimitiveName = field.isEmbedded() && field instanceof EncodedDataTypeSchema;
+            final boolean usePrimitiveName = field.type().isEmbedded() && field.type() instanceof EncodedDataTypeSchema;
             final String typeName = usePrimitiveName ?
-                ((EncodedDataTypeSchema)field).primitiveType().primitiveName() :
-                requireNonNull(typeToName.get(field));
+                ((EncodedDataTypeSchema)field.type()).primitiveType().primitiveName() :
+                requireNonNull(typeToName.get(field.type()));
 
             final Element element = document.createElement("field");
             element.setAttribute("id", Integer.toString(id));
             element.setAttribute("name", "member" + id);
             element.setAttribute("type", typeName);
+            element.setAttribute("presence", field.presence().name().toLowerCase());
             parent.appendChild(element);
         }
 
@@ -309,17 +312,35 @@ private static Element createTypeElement(
         final Document document,
         final String name,
         final String primitiveType,
-        final int length)
+        final int length,
+        final Encoding.Presence presence)
     {
-        final Element blockLength = document.createElement("type");
-        blockLength.setAttribute("name", name);
-        blockLength.setAttribute("primitiveType", primitiveType);
+        final Element typeElement = document.createElement("type");
+        typeElement.setAttribute("name", name);
+        typeElement.setAttribute("primitiveType", primitiveType);
+
         if (length > 1)
         {
-            blockLength.setAttribute("length", Integer.toString(length));
+            typeElement.setAttribute("length", Integer.toString(length));
         }
 
-        return blockLength;
+        switch (presence)
+        {
+
+            case REQUIRED:
+                typeElement.setAttribute("presence", "required");
+                break;
+            case OPTIONAL:
+                typeElement.setAttribute("presence", "optional");
+                break;
+            case CONSTANT:
+                typeElement.setAttribute("presence", "constant");
+                break;
+            default:
+                throw new IllegalArgumentException("Unknown presence: " + presence);
+        }
+
+        return typeElement;
     }
 
     private static Element createRefElement(
@@ -350,7 +371,13 @@ private static void appendTypes(
 
         for (final GroupSchema group : groups)
         {
-            appendTypes(visitedTypes, topLevelTypes, typeSchemaConverter, group.blockFields(), group.groups());
+            appendTypes(
+                visitedTypes,
+                topLevelTypes,
+                typeSchemaConverter,
+                group.blockFields().stream().map(FieldSchema::type).collect(Collectors.toList()),
+                group.groups()
+            );
         }
     }
 
@@ -380,7 +407,8 @@ public void onEncoded(final EncodedDataTypeSchema type)
                 document,
                 typeToName.computeIfAbsent(type, nextName),
                 type.primitiveType().primitiveName(),
-                type.length()
+                type.length(),
+                type.presence()
             );
         }