Support file pruning through JsonPredicates and predicates queryRequest parameters

server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java

@@ -33,17 +33,21 @@ public static ReadTableRequest api2ReadTableRequest(QueryRequest request) {
       throw new IllegalArgumentException("version cannot be negative.");
     } else if (request.getVersion() != null && request.getTimestamp() == null) {
       return new ReadTableRequest.ReadTableVersion(
-          request.getPredicateHints(),
+          Optional.ofNullable(request.getPredicateHints()),
+          Optional.ofNullable(request.getJsonPredicateHints()),
           Optional.ofNullable(request.getLimitHint()),
           request.getVersion());
     } else if (request.getVersion() == null && request.getTimestamp() != null) {
       return new ReadTableRequest.ReadTableAsOfTimestamp(
-          request.getPredicateHints(),
+          Optional.ofNullable(request.getPredicateHints()),
+          Optional.ofNullable(request.getJsonPredicateHints()),
           Optional.ofNullable(request.getLimitHint()),
           CommonMappers.parseTimestamp(request.getTimestamp()));
     } else if (request.getVersion() == null && request.getTimestamp() == null) {
       return new ReadTableRequest.ReadTableCurrentVersion(
-          request.getPredicateHints(), Optional.ofNullable(request.getLimitHint()));
+          Optional.ofNullable(request.getPredicateHints()),
+          Optional.ofNullable(request.getJsonPredicateHints()),
+          Optional.ofNullable(request.getLimitHint()));
     } else {
       throw new IllegalArgumentException("Cannot specify both version and timestamp");
     }

server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java

@@ -369,6 +369,54 @@ public void queryNotExistingTable() throws IOException {
         .statusCode(404);
   }
 
+  @DisabledOnOs(OS.WINDOWS)
+  @Test
+  public void queryTableCurrentVersionWithPredicates() throws IOException {
+    var responseBodyLines = given()
+        .when()
+        .filter(deltaFilter)
+        .body("{\"jsonPredicateHints\": \"{" + "  \\\"op\\\": \\\"equal\\\","
+            + "  \\\"children\\\": ["
+            + "    {\\\"op\\\": \\\"column\\\", \\\"name\\\":\\\"date\\\", \\\"valueType\\\":\\\"date\\\"},"
+            + "    {\\\"op\\\":\\\"literal\\\",\\\"value\\\":\\\"2021-04-29\\\",\\\"valueType\\\":\\\"date\\\"}"
+            + "  ]"
+            + "}\"}")
+        .header(new Header("Content-Type", "application/json"))
+        .post(
+            "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/query",
+            "name",
+            "default",
+            "table1")
+        .then()
+        .statusCode(200)
+        .extract()
+        .body()
+        .asString()
+        .split("\n");
+
+    assertEquals(
+        deltaTable1Protocol,
+        objectMapper.reader().readValue(responseBodyLines[0], ProtocolObject.class));
+    assertEquals(
+        deltaTable1Metadata,
+        objectMapper.reader().readValue(responseBodyLines[1], MetadataObject.class));
+    var files = Arrays.stream(responseBodyLines)
+        .skip(2)
+        .map(line -> {
+          try {
+            return objectMapper
+                .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+                .reader()
+                .readValue(line, FileObjectWithoutPresignedUrl.class);
+          } catch (IOException e) {
+            throw new RuntimeException(e);
+          }
+        })
+        .collect(Collectors.toSet());
+    assertEquals(7, responseBodyLines.length);
+    assertEquals(deltaTable1FilesWithoutPresignedUrl, files);
+  }
+
   @DisabledOnOs(OS.WINDOWS)
   @Test
   public void queryTableCurrentVersion() throws IOException {

server/core/build.gradle.kts

@@ -43,6 +43,9 @@ dependencies {
     compileOnly(String.format("com.amazonaws:aws-java-sdk-s3:%s", awsSdkVersion))
     implementation(String.format("org.apache.hadoop:hadoop-aws:%s", hadoopVersion))
 
+    //PREDICATE PARSER
+    implementation("com.github.jsqlparser:jsqlparser:4.8")
+
     // TEST
     testImplementation("org.junit.jupiter:junit-jupiter")
     testImplementation("io.quarkus:quarkus-arc")

server/core/src/main/java/io/whitefox/core/ColumnRange.java

@@ -0,0 +1,132 @@
+package io.whitefox.core;
+
+import io.whitefox.annotations.SkipCoverageGenerated;
+import io.whitefox.core.types.*;
+import io.whitefox.core.types.predicates.TypeNotSupportedException;
+import java.sql.Date;
+import java.sql.Timestamp;
+
+public class ColumnRange {
+
+  String minVal;
+  String maxVal;
+
+  DataType valueType;
+
+  public ColumnRange(String minVal, String maxVal, DataType valueType) {
+    this.minVal = minVal;
+    this.maxVal = maxVal;
+    this.valueType = valueType;
+  }
+
+  public ColumnRange(String onlyVal, DataType valueType) {
+    this.minVal = onlyVal;
+    this.maxVal = onlyVal;
+    this.valueType = valueType;
+  }
+
+  public DataType getValueType() {
+    return valueType;
+  }
+
+  public String getSingleValue() {
+    return minVal;
+  }
+
+  private Boolean typedContains(String point) throws TypeNotSupportedException {
+    if (valueType instanceof IntegerType) {
+      var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point));
+      var c2 = Integer.compare(Integer.parseInt(maxVal), Integer.parseInt(point));
+      return (c1 <= 0 && c2 >= 0);
+    } else if (valueType instanceof LongType) {
+      var c1 = Long.compare(Long.parseLong(minVal), Long.parseLong(point));
+      var c2 = Long.compare(Long.parseLong(maxVal), Long.parseLong(point));
+      return (c1 <= 0 && c2 >= 0);
+    } else if (valueType instanceof TimestampType) {
+      var c1 = Timestamp.valueOf(minVal).before(Timestamp.valueOf(point));
+      var c2 = Timestamp.valueOf(maxVal).after(Timestamp.valueOf(point));
+      return (c1 && c2) || Timestamp.valueOf(minVal).equals(Timestamp.valueOf(point));
+    } else if (valueType instanceof FloatType) {
+      var c1 = Float.compare(Float.parseFloat(minVal), Float.parseFloat(point));
+      var c2 = Float.compare(Float.parseFloat(maxVal), Float.parseFloat(point));
+      return (c1 <= 0 && c2 >= 0);
+    } else if (valueType instanceof DoubleType) {
+      var c1 = Double.compare(Double.parseDouble(minVal), Double.parseDouble(point));
+      var c2 = Double.compare(Double.parseDouble(maxVal), Double.parseDouble(point));
+      return (c1 <= 0 && c2 >= 0);
+    } else if (valueType instanceof DateType) {
+      var c1 = Date.valueOf(minVal).before(Date.valueOf(point));
+      var c2 = Date.valueOf(maxVal).after(Date.valueOf(point));
+      return (c1 && c2) || Date.valueOf(minVal).equals(Date.valueOf(point));
+    } else if (valueType instanceof BooleanType) {
+      var c1 = Boolean.parseBoolean(minVal) == Boolean.parseBoolean(point);
+      var c2 = Boolean.parseBoolean(maxVal) == Boolean.parseBoolean(point);
+      return c1 || c2;
+    } else if (valueType instanceof StringType) {
+      var c1 = minVal.compareTo(point);
+      var c2 = maxVal.compareTo(point);
+      return (c1 <= 0 && c2 >= 0);
+    } else throw new TypeNotSupportedException(valueType);
+  }
+
+  private Boolean typedLessThan(String point) throws TypeNotSupportedException {
+    if (valueType instanceof IntegerType) {
+      var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point));
+      return (c1 < 0);
+    } else if (valueType instanceof LongType) {
+      var c1 = Long.compare(Long.parseLong(minVal), Long.parseLong(point));
+      return (c1 < 0);
+    } else if (valueType instanceof TimestampType) {
+      return Timestamp.valueOf(minVal).before(Timestamp.valueOf(point));
+    } else if (valueType instanceof FloatType) {
+      var c1 = Float.compare(Float.parseFloat(minVal), Float.parseFloat(point));
+      return (c1 < 0);
+    } else if (valueType instanceof DoubleType) {
+      var c1 = Double.compare(Double.parseDouble(minVal), Double.parseDouble(point));
+      return (c1 < 0);
+    } else if (valueType instanceof DateType) {
+      return Date.valueOf(minVal).before(Date.valueOf(point));
+    } else if (valueType instanceof StringType) {
+      var c = minVal.compareTo(point);
+      return (c < 0);
+    } else throw new TypeNotSupportedException(valueType);
+  }
+
+  // not used currently
+  @SkipCoverageGenerated
+  private Boolean typedGreaterThan(String point) {
+    if (valueType instanceof IntegerType) {
+      var c = Integer.compare(Integer.parseInt(point), Integer.parseInt(maxVal));
+      return (c < 0);
+    } else if (valueType instanceof LongType) {
+      var c = Long.compare(Long.parseLong(point), Long.parseLong(maxVal));
+      return (c < 0);
+    } else if (valueType instanceof TimestampType) {
+      return Timestamp.valueOf(point).before(Timestamp.valueOf(maxVal));
+    } else if (valueType instanceof FloatType) {
+      var c = Float.compare(Float.parseFloat(maxVal), Float.parseFloat(point));
+      return (c < 0);
+    } else if (valueType instanceof DoubleType) {
+      var c = Double.compare(Double.parseDouble(maxVal), Double.parseDouble(point));
+      return (c < 0);
+    } else if (valueType instanceof DateType) {
+      return Date.valueOf(point).before(Date.valueOf(maxVal));
+
+    } else {
+      var c = point.compareTo(maxVal);
+      return (c < 0);
+    }
+  }
+
+  public Boolean contains(String point) throws TypeNotSupportedException {
+    return typedContains(point);
+  }
+
+  public Boolean canBeLess(String point) throws TypeNotSupportedException {
+    return typedLessThan(point);
+  }
+
+  public Boolean canBeGreater(String point) {
+    return typedGreaterThan(point);
+  }
+}

server/core/src/main/java/io/whitefox/core/DeltaObjectMapper.java

@@ -0,0 +1,25 @@
+package io.whitefox.core;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import io.whitefox.core.types.DataType;
+import io.whitefox.core.types.predicates.DataTypeDeserializer;
+
+public class DeltaObjectMapper {
+
+  private static final ObjectMapper objectMapper = newInstance();
+
+  private static ObjectMapper newInstance() {
+    ObjectMapper mapper = new ObjectMapper();
+    mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL);
+    var customSerializersModule = new SimpleModule();
+    customSerializersModule.addDeserializer(DataType.class, new DataTypeDeserializer());
+    mapper.registerModule(customSerializersModule);
+    return mapper;
+  }
+
+  public static ObjectMapper getInstance() {
+    return objectMapper;
+  }
+}

server/core/src/main/java/io/whitefox/core/FileStats.java

@@ -0,0 +1,50 @@
+package io.whitefox.core;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import java.util.Map;
+
+public class FileStats {
+  // {"numRecords":1,"minValues":{"id":0},"maxValues":{"id":0},"nullCount":{"id":0}}
+  @JsonProperty("numRecords")
+  String numRecords;
+
+  @JsonProperty("minValues")
+  Map<String, String> minValues;
+
+  @JsonProperty("maxValues")
+  Map<String, String> maxValues;
+
+  @JsonProperty("nullCount")
+  Map<String, String> nullCount;
+
+  public FileStats() {
+    super();
+  }
+
+  public String getNumRecords() {
+    return numRecords;
+  }
+
+  public Map<String, String> getMinValues() {
+    return minValues;
+  }
+
+  public Map<String, String> getMaxValues() {
+    return maxValues;
+  }
+
+  public Map<String, String> getNullCount() {
+    return nullCount;
+  }
+
+  public FileStats(
+      String numRecords,
+      Map<String, String> minValues,
+      Map<String, String> maxValues,
+      Map<String, String> nullCount) {
+    this.numRecords = numRecords;
+    this.minValues = minValues;
+    this.maxValues = maxValues;
+    this.nullCount = nullCount;
+  }
+}