[ADBDEV-5595] - Add support for new types in PXF

- Optimize imports
arenadata · Jul 11, 2024 · f8647c2 · f8647c2
1 parent e189002
commit f8647c2
Show file tree

Hide file tree

Showing 12 changed files with 5 additions and 55 deletions.
diff --git a/...f-hdfs/src/main/java/org/greenplum/pxf/plugins/hdfs/parquet/ParquetIntervalUtilities.java b/...f-hdfs/src/main/java/org/greenplum/pxf/plugins/hdfs/parquet/ParquetIntervalUtilities.java
@@ -2,10 +2,7 @@
 
 import lombok.experimental.UtilityClass;
 import lombok.extern.slf4j.Slf4j;
-import org.apache.parquet.example.data.simple.BinaryValue;
-import org.apache.parquet.example.data.simple.Primitive;
 import org.apache.parquet.io.api.Binary;
-import org.greenplum.pxf.api.io.DataType;
 import org.postgresql.util.PGInterval;
 
 import java.math.BigDecimal;

diff --git a/.../pxf-hdfs/src/main/java/org/greenplum/pxf/plugins/hdfs/parquet/ParquetOperatorPruner.java b/.../pxf-hdfs/src/main/java/org/greenplum/pxf/plugins/hdfs/parquet/ParquetOperatorPruner.java
@@ -1,13 +1,8 @@
 package org.greenplum.pxf.plugins.hdfs.parquet;
 
-import org.apache.parquet.schema.GroupType;
 import org.apache.parquet.schema.PrimitiveType;
 import org.apache.parquet.schema.Type;
-import org.greenplum.pxf.api.filter.ColumnIndexOperandNode;
-import org.greenplum.pxf.api.filter.Node;
-import org.greenplum.pxf.api.filter.Operator;
-import org.greenplum.pxf.api.filter.OperatorNode;
-import org.greenplum.pxf.api.filter.SupportedOperatorPruner;
+import org.greenplum.pxf.api.filter.*;
 import org.greenplum.pxf.api.utilities.ColumnDescriptor;
 
 import java.util.EnumSet;

diff --git a/...hdfs/src/main/java/org/greenplum/pxf/plugins/hdfs/parquet/ParquetRecordFilterBuilder.java b/...hdfs/src/main/java/org/greenplum/pxf/plugins/hdfs/parquet/ParquetRecordFilterBuilder.java
@@ -5,44 +5,21 @@
 import org.apache.parquet.filter2.predicate.FilterPredicate;
 import org.apache.parquet.filter2.predicate.Operators;
 import org.apache.parquet.io.api.Binary;
-import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.apache.parquet.schema.PrimitiveType;
 import org.apache.parquet.schema.Type;
-import org.greenplum.pxf.api.error.UnsupportedTypeException;
-import org.greenplum.pxf.api.filter.ColumnIndexOperandNode;
-import org.greenplum.pxf.api.filter.Node;
-import org.greenplum.pxf.api.filter.OperandNode;
-import org.greenplum.pxf.api.filter.Operator;
-import org.greenplum.pxf.api.filter.OperatorNode;
-import org.greenplum.pxf.api.filter.TreeVisitor;
+import org.greenplum.pxf.api.filter.*;
 import org.greenplum.pxf.api.utilities.ColumnDescriptor;
-import org.greenplum.pxf.plugins.hdfs.ParquetResolver;
 import org.greenplum.pxf.plugins.hdfs.parquet.converters.ParquetTypeConverter;
-import org.greenplum.pxf.plugins.hdfs.utilities.DecimalOverflowOption;
-import org.greenplum.pxf.plugins.hdfs.utilities.DecimalUtilities;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.time.LocalDate;
-import java.time.temporal.ChronoUnit;
 import java.util.Deque;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.function.BiFunction;
 
-import static org.apache.parquet.filter2.predicate.FilterApi.and;
-import static org.apache.parquet.filter2.predicate.FilterApi.binaryColumn;
-import static org.apache.parquet.filter2.predicate.FilterApi.booleanColumn;
-import static org.apache.parquet.filter2.predicate.FilterApi.doubleColumn;
-import static org.apache.parquet.filter2.predicate.FilterApi.floatColumn;
-import static org.apache.parquet.filter2.predicate.FilterApi.intColumn;
-import static org.apache.parquet.filter2.predicate.FilterApi.longColumn;
-import static org.apache.parquet.filter2.predicate.FilterApi.not;
-import static org.apache.parquet.filter2.predicate.FilterApi.or;
-import static org.apache.parquet.schema.LogicalTypeAnnotation.DateLogicalTypeAnnotation;
-import static org.apache.parquet.schema.LogicalTypeAnnotation.TimestampLogicalTypeAnnotation;
-import static org.apache.parquet.schema.LogicalTypeAnnotation.DecimalLogicalTypeAnnotation;
+import static org.apache.parquet.filter2.predicate.FilterApi.*;
 import static org.greenplum.pxf.plugins.hdfs.parquet.converters.ParquetTypeConverter.FILTER_COLUMN;
 
 /**

diff --git a/...-hdfs/src/main/java/org/greenplum/pxf/plugins/hdfs/parquet/ParquetTimestampUtilities.java b/...-hdfs/src/main/java/org/greenplum/pxf/plugins/hdfs/parquet/ParquetTimestampUtilities.java
@@ -1,6 +1,7 @@
 package org.greenplum.pxf.plugins.hdfs.parquet;
 
 import org.apache.parquet.example.data.simple.NanoTime;
+import org.apache.parquet.io.api.Binary;
 import org.apache.parquet.schema.LogicalTypeAnnotation;
 import org.greenplum.pxf.api.GreenplumDateTime;
 import org.slf4j.Logger;
@@ -11,8 +12,6 @@
 import java.time.*;
 import java.util.Base64;
 
-import org.apache.parquet.io.api.Binary;
-
 import static org.greenplum.pxf.plugins.hdfs.parquet.ParquetConstant.*;
 
 public class ParquetTimestampUtilities {

diff --git a/...n/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/BooleanParquetTypeConverter.java b/...n/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/BooleanParquetTypeConverter.java
@@ -2,8 +2,6 @@
 
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import org.apache.parquet.example.data.Group;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
 import org.greenplum.pxf.api.io.DataType;
 
 public class BooleanParquetTypeConverter implements ParquetTypeConverter {

diff --git a/...in/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/DoubleParquetTypeConverter.java b/...in/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/DoubleParquetTypeConverter.java
@@ -2,8 +2,6 @@
 
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import org.apache.parquet.example.data.Group;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
 import org.greenplum.pxf.api.io.DataType;
 
 public class DoubleParquetTypeConverter implements ParquetTypeConverter {

diff --git a/.../greenplum/pxf/plugins/hdfs/parquet/converters/FixedLenByteArrayParquetTypeConverter.java b/.../greenplum/pxf/plugins/hdfs/parquet/converters/FixedLenByteArrayParquetTypeConverter.java
@@ -1,7 +1,6 @@
 package org.greenplum.pxf.plugins.hdfs.parquet.converters;
 
 import com.fasterxml.jackson.databind.node.ArrayNode;
-import lombok.Data;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.DecoderException;
 import org.apache.parquet.example.data.Group;
@@ -16,8 +15,6 @@
 
 import java.math.BigDecimal;
 import java.math.BigInteger;
-import java.nio.ByteBuffer;
-import java.util.UUID;
 
 import static org.greenplum.pxf.plugins.hdfs.parquet.ParquetIntervalUtilities.INTERVAL_TYPE_LENGTH;
 
@@ -27,7 +24,7 @@ public class FixedLenByteArrayParquetTypeConverter implements ParquetTypeConvert
     private final Type type;
     private final DataType dataType;
     private final DataType detectedDataType;
-    private DecimalUtilities decimalUtilities;
+    private final DecimalUtilities decimalUtilities;
 
     public FixedLenByteArrayParquetTypeConverter(Type type, DataType dataType, DecimalUtilities decimalUtilities) {
         this.type = type;

diff --git a/...ain/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/FloatParquetTypeConverter.java b/...ain/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/FloatParquetTypeConverter.java
@@ -2,8 +2,6 @@
 
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import org.apache.parquet.example.data.Group;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
 import org.greenplum.pxf.api.io.DataType;
 
 public class FloatParquetTypeConverter implements ParquetTypeConverter {

diff --git a/...ain/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/Int32ParquetTypeConverter.java b/...ain/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/Int32ParquetTypeConverter.java
@@ -3,7 +3,6 @@
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import org.apache.parquet.example.data.Group;
 import org.apache.parquet.schema.LogicalTypeAnnotation;
-import org.apache.parquet.schema.PrimitiveType;
 import org.apache.parquet.schema.Type;
 import org.greenplum.pxf.api.GreenplumDateTime;
 import org.greenplum.pxf.api.io.DataType;
@@ -17,7 +16,6 @@
 import java.time.temporal.ChronoField;
 import java.util.Objects;
 
-import static org.greenplum.pxf.api.GreenplumDateTime.NANOS_IN_MICROS;
 import static org.greenplum.pxf.api.GreenplumDateTime.NANOS_IN_MILLIS;
 
 public class Int32ParquetTypeConverter implements ParquetTypeConverter {

diff --git a/...ain/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/Int64ParquetTypeConverter.java b/...ain/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/Int64ParquetTypeConverter.java
@@ -3,7 +3,6 @@
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import org.apache.parquet.example.data.Group;
 import org.apache.parquet.schema.LogicalTypeAnnotation;
-import org.apache.parquet.schema.PrimitiveType;
 import org.apache.parquet.schema.Type;
 import org.greenplum.pxf.api.GreenplumDateTime;
 import org.greenplum.pxf.api.io.DataType;
@@ -17,7 +16,6 @@
 import java.time.temporal.ChronoField;
 
 import static org.greenplum.pxf.api.GreenplumDateTime.NANOS_IN_MICROS;
-import static org.greenplum.pxf.api.GreenplumDateTime.NANOS_IN_MILLIS;
 import static org.greenplum.pxf.plugins.hdfs.ParquetResolver.TIMESTAMP_PATTERN;
 
 public class Int64ParquetTypeConverter implements ParquetTypeConverter {

diff --git a/...ain/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/Int96ParquetTypeConverter.java b/...ain/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/Int96ParquetTypeConverter.java
@@ -2,10 +2,7 @@
 
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import org.apache.parquet.example.data.Group;
-import org.apache.parquet.schema.PrimitiveType;
-import org.apache.parquet.schema.Type;
 import org.greenplum.pxf.api.io.DataType;
-import org.greenplum.pxf.plugins.hdfs.ParquetResolver;
 import org.greenplum.pxf.plugins.hdfs.parquet.ParquetTimestampUtilities;
 
 import static org.greenplum.pxf.plugins.hdfs.ParquetResolver.TIMESTAMP_PATTERN;

diff --git a/...src/main/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/ParquetTypeConverter.java b/...src/main/java/org/greenplum/pxf/plugins/hdfs/parquet/converters/ParquetTypeConverter.java
@@ -3,9 +3,7 @@
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import org.apache.commons.codec.DecoderException;
 import org.apache.commons.codec.binary.Hex;
-import lombok.extern.slf4j.Slf4j;
 import org.apache.parquet.example.data.Group;
-import org.apache.parquet.schema.Type;
 import org.greenplum.pxf.api.io.DataType;
 
 import java.nio.charset.StandardCharsets;