DRILL-6588: Make Sys tables of nullable datatypes

This is to address the generic problem of NULL values being projected as a string because of all datatypes being non-nullable.
This patch only applies to tables backed by the PojoReader (in our case, System tables). Added NonNullable annotations wherever application in any of the System tables, along with a unit test that verifies both nullable and non-nullable datatypes exist in the system tables.

closes #1371

exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordDataType.java

@@ -17,22 +17,23 @@
  */
 package org.apache.drill.exec.store;
 
-import com.google.common.collect.Lists;
+import java.util.AbstractMap.SimpleImmutableEntry;
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.calcite.rel.type.RelDataType;
 import org.apache.calcite.rel.type.RelDataTypeFactory;
 import org.apache.calcite.sql.type.SqlTypeName;
 
-import java.util.List;
-
 /**
  * RecordDataType defines names and data types of columns in a static drill table.
  */
 public abstract class RecordDataType {
 
   /**
-   * @return the {@link org.apache.calcite.sql.type.SqlTypeName} of columns in the table
+   * @return the {@link org.apache.calcite.sql.type.SqlTypeName} of columns in the table as a pair with its nullability
    */
-  public abstract List<SqlTypeName> getFieldSqlTypeNames();
+  public abstract List<SimpleImmutableEntry<SqlTypeName, Boolean>> getFieldSqlTypeNames();
 
   /**
    * @return the column names in the table
@@ -47,17 +48,21 @@ public abstract class RecordDataType {
    * @return the constructed type
    */
   public final RelDataType getRowType(RelDataTypeFactory factory) {
-    final List<SqlTypeName> types = getFieldSqlTypeNames();
+    final List<SimpleImmutableEntry<SqlTypeName, Boolean>> types = getFieldSqlTypeNames();
     final List<String> names = getFieldNames();
-    final List<RelDataType> fields = Lists.newArrayList();
-    for (final SqlTypeName typeName : types) {
+    final List<RelDataType> fields = new ArrayList<>();
+    for (SimpleImmutableEntry<SqlTypeName, Boolean> sqlTypePair : types) {
+      final SqlTypeName typeName = sqlTypePair.getKey();
+      final RelDataType tempDataType;
       switch (typeName) {
         case VARCHAR:
-          fields.add(factory.createSqlType(typeName, Integer.MAX_VALUE));
+          tempDataType = factory.createSqlType(typeName, Integer.MAX_VALUE);
           break;
         default:
-          fields.add(factory.createSqlType(typeName));
+          tempDataType = factory.createSqlType(typeName);
       }
+      //Add [Non]Nullable RelDataType
+      fields.add(factory.createTypeWithNullability(tempDataType, sqlTypePair.getValue()));
     }
     return factory.createStructType(fields, names);
   }

exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/NonNullable.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.store.pojo;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Indicates NonNullable nature
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.FIELD)
+public @interface NonNullable {
+  boolean value() default true;
+}

exec/java-exec/src/main/java/org/apache/drill/exec/store/pojo/PojoDataType.java

@@ -21,11 +21,11 @@
 import java.lang.reflect.Modifier;
 import java.math.BigDecimal;
 import java.sql.Timestamp;
+import java.util.AbstractMap.SimpleImmutableEntry;
+import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.calcite.sql.type.SqlTypeName;
-
-import com.google.common.collect.Lists;
 import org.apache.drill.exec.store.RecordDataType;
 
 /**
@@ -34,8 +34,8 @@
 public class PojoDataType extends RecordDataType {
 //  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(PojoDataType.class);
 
-  private final List<SqlTypeName> types = Lists.newArrayList();
-  private final List<String> names = Lists.newArrayList();
+  private final List<SimpleImmutableEntry<SqlTypeName, Boolean>> types = new ArrayList<>();
+  private final List<String> names = new ArrayList<>();
   private final Class<?> pojoClass;
 
   public PojoDataType(Class<?> pojoClass) {
@@ -48,22 +48,25 @@ public PojoDataType(Class<?> pojoClass) {
       Class<?> type = f.getType();
       names.add(f.getName());
 
+      //Absence of annotation @NonNullable => (isNullable=true)
+      final boolean isNullable = !(f.isAnnotationPresent(NonNullable.class));
+
       if (type == int.class || type == Integer.class) {
-        types.add(SqlTypeName.INTEGER);
+        types.add(new SimpleImmutableEntry<SqlTypeName, Boolean>(SqlTypeName.INTEGER, isNullable));
       } else if(type == boolean.class || type == Boolean.class) {
-        types.add(SqlTypeName.BOOLEAN);
+        types.add(new SimpleImmutableEntry<SqlTypeName, Boolean>(SqlTypeName.BOOLEAN, isNullable));
       } else if(type == long.class || type == Long.class) {
-        types.add(SqlTypeName.BIGINT);
+        types.add(new SimpleImmutableEntry<SqlTypeName, Boolean>(SqlTypeName.BIGINT, isNullable));
       } else if(type == double.class || type == Double.class) {
-        types.add(SqlTypeName.DOUBLE);
+        types.add(new SimpleImmutableEntry<SqlTypeName, Boolean>(SqlTypeName.DOUBLE, isNullable));
       } else if(type == BigDecimal.class) {
-        types.add(SqlTypeName.DECIMAL);
+        types.add(new SimpleImmutableEntry<SqlTypeName, Boolean>(SqlTypeName.DECIMAL, isNullable));
       } else if(type == String.class) {
-        types.add(SqlTypeName.VARCHAR);
+        types.add(new SimpleImmutableEntry<SqlTypeName, Boolean>(SqlTypeName.VARCHAR, isNullable));
       } else if(type.isEnum()) {
-        types.add(SqlTypeName.VARCHAR);
+        types.add(new SimpleImmutableEntry<SqlTypeName, Boolean>(SqlTypeName.VARCHAR, isNullable));
       } else if (type == Timestamp.class) {
-        types.add(SqlTypeName.TIMESTAMP);
+        types.add(new SimpleImmutableEntry<SqlTypeName, Boolean>(SqlTypeName.TIMESTAMP, isNullable));
       } else {
         throw new RuntimeException(String.format("PojoDataType doesn't yet support conversions from type [%s].", type));
       }
@@ -75,13 +78,12 @@ public Class<?> getPojoClass() {
   }
 
   @Override
-  public List<SqlTypeName> getFieldSqlTypeNames() {
+  public List<SimpleImmutableEntry<SqlTypeName, Boolean>> getFieldSqlTypeNames() {
     return types;
   }
 
   @Override
   public List<String> getFieldNames() {
     return names;
   }
-
 }

exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/BitToUserConnectionIterator.java

@@ -21,8 +21,8 @@
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
-import java.util.Set;
 import java.util.Map.Entry;
+import java.util.Set;
 import java.util.TimeZone;
 
 import org.apache.drill.exec.ExecConstants;
@@ -32,6 +32,7 @@
 import org.apache.drill.exec.rpc.user.UserSession;
 import org.apache.drill.exec.server.options.OptionManager;
 import org.apache.drill.exec.server.rest.profile.SimpleDurationFormat;
+import org.apache.drill.exec.store.pojo.NonNullable;
 import org.apache.drill.exec.util.ImpersonationUtil;
 import org.joda.time.DateTime;
 
@@ -99,14 +100,18 @@ public void remove() {
 
   public static class ConnectionInfo {
     public String user;
+    @NonNullable
     public String client;
+    @NonNullable
     public String drillbit;
+    @NonNullable
     public Timestamp established;
     public String duration;
     public int queries;
     public boolean isAuthenticated;
     public boolean isEncrypted;
     public boolean usingSSL;
+    @NonNullable
     public String session;
 
     public ConnectionInfo(Entry<BitToUserConnection, BitToUserConnectionConfig> connectionConfigPair, String hostname) {

exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/DrillbitIterator.java

@@ -21,6 +21,7 @@
 
 import org.apache.drill.exec.ops.ExecutorFragmentContext;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.store.pojo.NonNullable;
 
 public class DrillbitIterator implements Iterator<Object> {
   static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(DrillbitIterator.class);
@@ -34,13 +35,17 @@ public DrillbitIterator(ExecutorFragmentContext c) {
   }
 
   public static class DrillbitInstance {
+    @NonNullable
     public String hostname;
     public int user_port;
     public int control_port;
     public int data_port;
     public int http_port;
+    @NonNullable
     public boolean current;
+    @NonNullable
     public String version;
+    @NonNullable
     public String state;
   }
 

exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ExtendedOptionIterator.java

@@ -24,13 +24,14 @@
 import java.util.List;
 import java.util.Map;
 
-
-import com.google.common.collect.Lists;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.server.options.OptionManager;
 import org.apache.drill.exec.server.options.OptionValue;
 import org.apache.drill.exec.server.options.OptionValue.Kind;
 import org.apache.drill.exec.server.options.OptionValue.OptionScope;
+import org.apache.drill.exec.store.pojo.NonNullable;
+
+import com.google.common.collect.Lists;
 
 /*
  * Extends the original Option iterator. The idea is to hide the implementation details and present the
@@ -134,10 +135,14 @@ public enum Status {
    */
   public static class ExtendedOptionValueWrapper {
 
+    @NonNullable
     public final String name;
+    @NonNullable
     public final String kind;
+    @NonNullable
     public final OptionValue.AccessibleScopes accessibleScopes;
     public final String val;
+    @NonNullable
     public final OptionScope optionScope;
 
 

exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/MemoryIterator.java

@@ -27,6 +27,7 @@
 import org.apache.drill.common.config.DrillConfig;
 import org.apache.drill.exec.ops.ExecutorFragmentContext;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.store.pojo.NonNullable;
 
 public class MemoryIterator implements Iterator<Object> {
 
@@ -83,6 +84,7 @@ public void remove() {
   }
 
   public static class MemoryInfo {
+    @NonNullable
     public String hostname;
     public long user_port;
     public long heap_current;

exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/OptionIterator.java

@@ -21,15 +21,17 @@
 import java.util.Iterator;
 import java.util.List;
 
-import com.google.common.collect.Iterators;
-import com.google.common.collect.Lists;
 import org.apache.drill.exec.ops.FragmentContext;
 import org.apache.drill.exec.server.options.DrillConfigIterator;
 import org.apache.drill.exec.server.options.OptionManager;
 import org.apache.drill.exec.server.options.OptionValue;
+import org.apache.drill.exec.server.options.OptionValue.AccessibleScopes;
 import org.apache.drill.exec.server.options.OptionValue.Kind;
 import org.apache.drill.exec.server.options.OptionValue.OptionScope;
-import org.apache.drill.exec.server.options.OptionValue.AccessibleScopes;
+import org.apache.drill.exec.store.pojo.NonNullable;
+
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
 
 public class OptionIterator implements Iterator<Object> {
 //  private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(OptionIterator.class);
@@ -104,10 +106,15 @@ public enum Status {
    */
   public static class OptionValueWrapper {
 
+    @NonNullable
     public final String name;
+    @NonNullable
     public final Kind kind;
+    @NonNullable
     public final AccessibleScopes accessibleScopes;
+    @NonNullable
     public final OptionScope optionScope;
+    @NonNullable
     public final Status status;
     public final Long num_val;
     public final String string_val;

exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileInfoIterator.java

@@ -17,17 +17,19 @@
  */
 package org.apache.drill.exec.store.sys;
 
-import com.google.common.base.Function;
-import com.google.common.collect.Iterators;
+import java.sql.Timestamp;
+import java.util.Iterator;
+import java.util.Map.Entry;
+
+import javax.annotation.Nullable;
 
 import org.apache.drill.exec.ops.ExecutorFragmentContext;
 import org.apache.drill.exec.proto.UserBitShared;
 import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
+import org.apache.drill.exec.store.pojo.NonNullable;
 
-import javax.annotation.Nullable;
-import java.sql.Timestamp;
-import java.util.Iterator;
-import java.util.Map.Entry;
+import com.google.common.base.Function;
+import com.google.common.collect.Iterators;
 
 /**
  * System table listing completed profiles
@@ -114,8 +116,10 @@ public static class ProfileInfo {
 
     private static final ProfileInfo DEFAULT = new ProfileInfo();
 
+    @NonNullable
     public final String queryId;
     public final Timestamp startTime;
+    @NonNullable
     public final String foreman;
     public final long fragments;
     public final String user;

exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ProfileJsonIterator.java

@@ -27,6 +27,7 @@
 import org.apache.drill.exec.proto.UserBitShared;
 import org.apache.drill.exec.proto.UserBitShared.QueryProfile;
 import org.apache.drill.exec.serialization.InstanceSerializer;
+import org.apache.drill.exec.store.pojo.NonNullable;
 
 import com.google.common.base.Function;
 import com.google.common.collect.Iterators;
@@ -114,6 +115,7 @@ public static class ProfileJson {
 
     private static final ProfileJson DEFAULT = new ProfileJson();
 
+    @NonNullable
     public final String queryId;
     public final String json;
 

exec/java-exec/src/main/java/org/apache/drill/exec/store/sys/ThreadsIterator.java

@@ -23,6 +23,7 @@
 
 import org.apache.drill.exec.ops.ExecutorFragmentContext;
 import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
+import org.apache.drill.exec.store.pojo.NonNullable;
 
 public class ThreadsIterator implements Iterator<Object> {
 
@@ -62,6 +63,7 @@ public void remove() {
   }
 
   public static class ThreadsInfo {
+    @NonNullable
     public String hostname;
     public long user_port;
     public long total_threads;