From 0f0a847a86a6e5d8787a34e91bc72cebba3ade76 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 3 Nov 2023 18:55:29 +0100 Subject: [PATCH 01/30] wip-first draft of JsonPredicates and a test partitioned table --- .../core/services/DeltaSharedTable.java | 12 + .../core/services/DeltaSharedTableTest.java | 25 ++ .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 36 bytes .../_delta_log/00000000000000000000.json | 12 + ...-80ae-e8fe85adadf6.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-9c0b-5d8b63a32bd2.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-aeef-38ece808b77f.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-ac90-bf9424a681bc.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-a923-016823f92121.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...41af-80ae-e8fe85adadf6.c000.snappy.parquet | Bin 0 -> 478 bytes ...402d-9c0b-5d8b63a32bd2.c000.snappy.parquet | Bin 0 -> 478 bytes ...4d2c-aeef-38ece808b77f.c000.snappy.parquet | Bin 0 -> 478 bytes ...4640-ac90-bf9424a681bc.c000.snappy.parquet | Bin 0 -> 478 bytes ...442e-a923-016823f92121.c000.snappy.parquet | Bin 0 -> 478 bytes ...-a0c5-9497e34cde7d.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-82e4-b521351e5a8c.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-bfba-f238deee6371.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...-86f5-5de21ed67eee.c000.snappy.parquet.crc | Bin 0 -> 12 bytes ...4863-a0c5-9497e34cde7d.c000.snappy.parquet | Bin 0 -> 478 bytes ...4f09-82e4-b521351e5a8c.c000.snappy.parquet | Bin 0 -> 478 bytes ...4623-bfba-f238deee6371.c000.snappy.parquet | Bin 0 -> 478 bytes ...465d-86f5-5de21ed67eee.c000.snappy.parquet | Bin 0 -> 478 bytes .../java/io/whitefox/core/EvalHelper.java | 103 +++++ .../java/io/whitefox/core/JsonPredicates.java | 385 ++++++++++++++++++ .../whitefox/core/PartitionFilterUtils.java | 5 + 25 files changed, 542 insertions(+) create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/_delta_log/.00000000000000000000.json.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/_delta_log/00000000000000000000.json create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00001-ef9d73d7-88f2-41af-80ae-e8fe85adadf6.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00003-a29698f2-7d2c-402d-9c0b-5d8b63a32bd2.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00004-1602573a-de57-4d2c-aeef-38ece808b77f.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00006-8669602d-9175-4640-ac90-bf9424a681bc.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00007-2a965cbf-a36c-442e-a923-016823f92121.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00001-ef9d73d7-88f2-41af-80ae-e8fe85adadf6.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00003-a29698f2-7d2c-402d-9c0b-5d8b63a32bd2.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00004-1602573a-de57-4d2c-aeef-38ece808b77f.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00006-8669602d-9175-4640-ac90-bf9424a681bc.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00007-2a965cbf-a36c-442e-a923-016823f92121.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/.part-00009-28b727ce-c7d7-4863-a0c5-9497e34cde7d.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/.part-00011-ab578c13-0186-4f09-82e4-b521351e5a8c.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/.part-00013-def8488a-2bf2-4623-bfba-f238deee6371.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/.part-00015-7ca44de2-8b15-465d-86f5-5de21ed67eee.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/part-00009-28b727ce-c7d7-4863-a0c5-9497e34cde7d.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/part-00011-ab578c13-0186-4f09-82e4-b521351e5a8c.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/part-00013-def8488a-2bf2-4623-bfba-f238deee6371.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/part-00015-7ca44de2-8b15-465d-86f5-5de21ed67eee.c000.snappy.parquet create mode 100644 server/src/main/java/io/whitefox/core/EvalHelper.java create mode 100644 server/src/main/java/io/whitefox/core/JsonPredicates.java create mode 100644 server/src/main/java/io/whitefox/core/PartitionFilterUtils.java diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index 44f31e129..a32c77891 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -2,8 +2,12 @@ import io.delta.standalone.DeltaLog; import io.delta.standalone.Snapshot; +import io.delta.standalone.actions.AddFile; import io.whitefox.core.*; import java.sql.Timestamp; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; import java.util.Optional; import java.util.stream.Collectors; @@ -76,6 +80,14 @@ public Optional getTableVersion(Optional startingTimestamp) { return getSnapshot(startingTimestamp).map(Snapshot::getVersion); } + public boolean filterFilesBasedOnPredicates(List predicates, AddFile f) { + var partitionValues = f.getPartitionValues(); + predicates.forEach(p -> { + + }); + return true; + } + public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { Snapshot snapshot; if (readTableRequest instanceof ReadTableRequest.ReadTableCurrentVersion) { diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index 322b4f80c..f03fe3147 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -5,7 +5,11 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.wildfly.common.Assert.assertTrue; +import io.whitefox.core.Protocol; +import io.whitefox.core.ReadTableRequest; import io.whitefox.core.SharedTable; +import java.time.format.DateTimeParseException; +import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Test; @@ -61,4 +65,25 @@ void getTableVersionWithFutureTimestamp() throws ExecutionException, Interrupted var version = DTable.getTableVersion(TestDateUtils.parseTimestamp("2024-10-20T10:15:30+01:00")); assertEquals(Optional.empty(), version); } +<<<<<<< HEAD +======= + + @Test + void getTableVersionWithMalformedTimestamp() throws ExecutionException, InterruptedException { + var PTable = new SharedTable("delta-table", "default", "share1", deltaTable("delta-table")); + var DTable = DeltaSharedTable.of(PTable); + assertThrows( + DateTimeParseException.class, + () -> DTable.getTableVersion(Optional.of("221rfewdsad10:15:30+01:00"))); + } + + @Test + void queryTable() throws ExecutionException, InterruptedException { + var PTable = new SharedTable("partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); + var DTable = DeltaSharedTable.of(PTable); + var request = new ReadTableRequest.ReadTableCurrentVersion(List.of("date = '2021-08-09'"), Optional.empty()); + var response = DTable.queryTable(request); + assertTrue(2 == 2); + } +>>>>>>> 82552ab (wip-first draft of JsonPredicates and a test partitioned table) } diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/_delta_log/.00000000000000000000.json.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..614fc5eb7bc1703e1dcef608ddb5c52d19be1a4a GIT binary patch literal 36 rcmYc;N@ieSU}Dh8&Pwciq#fj8YEn|PDYqy5h?d&>#LSkekfu5S(fkcU literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/_delta_log/00000000000000000000.json b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/_delta_log/00000000000000000000.json new file mode 100644 index 000000000..18bd945b6 --- /dev/null +++ b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/_delta_log/00000000000000000000.json @@ -0,0 +1,12 @@ +{"commitInfo":{"timestamp":1698863537194,"operation":"WRITE","operationParameters":{"mode":"ErrorIfExists","partitionBy":"[\"date\"]"},"isolationLevel":"Serializable","isBlindAppend":true,"operationMetrics":{"numFiles":"9","numOutputRows":"9","numOutputBytes":"4302"},"engineInfo":"Apache-Spark/3.3.0 Delta-Lake/2.3.0","txnId":"bfc58406-b971-4499-8ae3-e61ff56c5c1f"}} +{"protocol":{"minReaderVersion":1,"minWriterVersion":2}} +{"metaData":{"id":"d07c494b-e59b-4848-9bcd-2a08b307bce1","format":{"provider":"parquet","options":{}},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"id\",\"type\":\"long\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"string\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["date"],"configuration":{},"createdTime":1698863528592}} +{"add":{"path":"date=2021-08-09/part-00001-ef9d73d7-88f2-41af-80ae-e8fe85adadf6.c000.snappy.parquet","partitionValues":{"date":"2021-08-09"},"size":478,"modificationTime":1698863536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":0},\"maxValues\":{\"id\":0},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"date=2021-08-09/part-00003-a29698f2-7d2c-402d-9c0b-5d8b63a32bd2.c000.snappy.parquet","partitionValues":{"date":"2021-08-09"},"size":478,"modificationTime":1698863536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":1},\"maxValues\":{\"id\":1},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"date=2021-08-09/part-00004-1602573a-de57-4d2c-aeef-38ece808b77f.c000.snappy.parquet","partitionValues":{"date":"2021-08-09"},"size":478,"modificationTime":1698863536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":2},\"maxValues\":{\"id\":2},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"date=2021-08-09/part-00006-8669602d-9175-4640-ac90-bf9424a681bc.c000.snappy.parquet","partitionValues":{"date":"2021-08-09"},"size":478,"modificationTime":1698863536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":3},\"maxValues\":{\"id\":3},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"date=2021-08-09/part-00007-2a965cbf-a36c-442e-a923-016823f92121.c000.snappy.parquet","partitionValues":{"date":"2021-08-09"},"size":478,"modificationTime":1698863536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":4},\"maxValues\":{\"id\":4},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"date=2021-08-15/part-00009-28b727ce-c7d7-4863-a0c5-9497e34cde7d.c000.snappy.parquet","partitionValues":{"date":"2021-08-15"},"size":478,"modificationTime":1698863536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":6},\"maxValues\":{\"id\":6},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"date=2021-08-15/part-00011-ab578c13-0186-4f09-82e4-b521351e5a8c.c000.snappy.parquet","partitionValues":{"date":"2021-08-15"},"size":478,"modificationTime":1698863536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":7},\"maxValues\":{\"id\":7},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"date=2021-08-15/part-00013-def8488a-2bf2-4623-bfba-f238deee6371.c000.snappy.parquet","partitionValues":{"date":"2021-08-15"},"size":478,"modificationTime":1698863536000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":8},\"maxValues\":{\"id\":8},\"nullCount\":{\"id\":0}}"}} +{"add":{"path":"date=2021-08-15/part-00015-7ca44de2-8b15-465d-86f5-5de21ed67eee.c000.snappy.parquet","partitionValues":{"date":"2021-08-15"},"size":478,"modificationTime":1698863537000,"dataChange":true,"stats":"{\"numRecords\":1,\"minValues\":{\"id\":9},\"maxValues\":{\"id\":9},\"nullCount\":{\"id\":0}}"}} diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00001-ef9d73d7-88f2-41af-80ae-e8fe85adadf6.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00001-ef9d73d7-88f2-41af-80ae-e8fe85adadf6.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..fe3a6258ebd3841a1c959e34fd6e1619a39c0e14 GIT binary patch literal 12 TcmYc;N@ieSU}AWaFDeHB62$_^ literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00003-a29698f2-7d2c-402d-9c0b-5d8b63a32bd2.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00003-a29698f2-7d2c-402d-9c0b-5d8b63a32bd2.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..d5e19e43ac1863f33ca5ef62161ee132253525b4 GIT binary patch literal 12 TcmYc;N@ieSU}9k7=Z^sZ4nG0A literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00004-1602573a-de57-4d2c-aeef-38ece808b77f.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00004-1602573a-de57-4d2c-aeef-38ece808b77f.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..40544516f505de526774dc841aae9a7e79afc09c GIT binary patch literal 12 TcmYc;N@ieSU}9hq;PVCm4krPq literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00006-8669602d-9175-4640-ac90-bf9424a681bc.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00006-8669602d-9175-4640-ac90-bf9424a681bc.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..caa0eea6ed9729a2ded81a9b1f23dabe433c7c7f GIT binary patch literal 12 TcmYc;N@ieSU}AVzAi@a%61f7l literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00007-2a965cbf-a36c-442e-a923-016823f92121.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/.part-00007-2a965cbf-a36c-442e-a923-016823f92121.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..1f3c5de55aac400c5cc19aa2bebf40f2b38f5c8f GIT binary patch literal 12 TcmYc;N@ieSU}8`;Qd$N850nC0 literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00001-ef9d73d7-88f2-41af-80ae-e8fe85adadf6.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00001-ef9d73d7-88f2-41af-80ae-e8fe85adadf6.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d6cec93b8e3ad5a56ff43ec84d3eb5ae95582739 GIT binary patch literal 478 zcmZWmO-sW-5S^~I4W><-q>7YY^e^~V zJo?*oTB<#`hh^TInfD&d;Oe$cfjZQsU!ULa#||Y33)BKQX%GN7$lLT&@eXOpC1JhKm# zRI&zk<=;fQh$LoTEKMep_(e`lrURX6d$)eQ*NV8<=XQC>`p^1W2^zK$-rl*@$ z7$ChHi{jY?(QFw-dK6(7*GyIX3X0xpy}n1QbE&3<+cr1}rVap~>`(2=2s1DK5GZk!dom!93bL6e->K4StK? z;L2U1XEEG!?&BOdldD^YAYu}We1Cp@9GL_ItPu^NlMY4*9aJjtZ=Z3B4PD!# zvLx7`&^DA)ngvO$ z9||sb8|kfo6WJ={;C)r7oTtePk5n#wnM-}TX8G$}!EIV*zDxB7i0 q+TNf&6k^a*Uf1=6Hw3dZ%P>ty;0;omK(are=VmPc literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00004-1602573a-de57-4d2c-aeef-38ece808b77f.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00004-1602573a-de57-4d2c-aeef-38ece808b77f.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c655515d2f7b6c1a8955af3c9e5dd847766af03c GIT binary patch literal 478 zcmZWm%SyvQ6uli=$wCC7GbE5j7_hWZhc?qUBDj+-N^#MRh)k1d4W_R=6e->K4gP>T z|HM!5cf66>ES$w~&pG$rbKs7zZ<`dTO&$96`TbF{DM5Hl4S>@Y0f3`I_GiyJBbI3# zP+n4EF_dj)&=D4ZxPEgIuT}K)5POe$p(p`bJSQdzd|AOw%Q>QD8ir~bqKad>&i@wj!FIR1-+?*Y+g&UoXP?h$il!@Er#el8(096B sEgN31(U-E<(O%p2q&HI?PqwCAJ(JRHO4s%Jt(s-qc!9S9z!)$41Ly)}$p8QV literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00006-8669602d-9175-4640-ac90-bf9424a681bc.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00006-8669602d-9175-4640-ac90-bf9424a681bc.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..32faa5bbc2fa25badf84d29fe30fffe31e7e93d7 GIT binary patch literal 478 zcmZWmO-sW-5S?zT4W?iDP^9!C{)GND z|BFYbQk#Q&SmwPqGw;ESZtoius7)RE`Srak+LRzXr3S!xlK?=mP{e=#tP5h9#u4Qu zB^E>3W(FN%0f^nZvt*^>m#4&k)^kM((9$I_Nf-blmTlP;!=@#xWx%PmXDMNX4cM?^ zT<3f9ib3^goU>?&Ssz9a69B+8~zS6wUlQho!l)HQy>L^Hf|6#)yYQC4k z0O>TZd>~;&zJsGwSC!~bo5k8uOU6p?=@?dZQ}(#3IHR#><{W_Wy}Bo literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00007-2a965cbf-a36c-442e-a923-016823f92121.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-09/part-00007-2a965cbf-a36c-442e-a923-016823f92121.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..80b30c4b8ddccfd82a55ddc6008a0dd5a4ab0878 GIT binary patch literal 478 zcmZWm%SyvQ6ulj5$wCC7GbE5j7_hVuhc?qUBDj+-N^#MRh)k1d4W_R=6e->K4StLG z8UBiY;H}hV;Vg!G&bjxV19xd=qxuTNrAg0MslfYT-cfTKbY|J}3Bh-DfF zl$VrP3}u@cbch8Yc5hCSm5QGp68}-p6(vB6=foso0E}3+WmgQF7Oj>6C2Pl0!U!9% zVa2%2_vRIY>cKc?(Gs(w9RPbT&@vm-E5I$JVz6Nu6S=0jjAufv6s8crfyA?plW@5( zkEM{J4rc9NN3xCtW?yGI74h;#%yp^)l`3<*;pyv2^FB|~IP+8P@@c4}AmRPH4PU7F zP6`90_ifQ~F+;S-qDW05RB=qZ{BI%eZ?@a}EtrG7-Nih1_W7KKXn4YNs*@~?0;ki_ rvhH{5Jt@0w?YBH%`cu{RWpmQeQz^ZM^gO@UtXZ~=7kDcGjPSBQd0=K7 literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/.part-00009-28b727ce-c7d7-4863-a0c5-9497e34cde7d.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/.part-00009-28b727ce-c7d7-4863-a0c5-9497e34cde7d.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..dcca5ede7405bd5e37415ee4499f7a79ed0a6359 GIT binary patch literal 12 TcmYc;N@ieSU}8AyYxx-f695B& literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/.part-00011-ab578c13-0186-4f09-82e4-b521351e5a8c.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/.part-00011-ab578c13-0186-4f09-82e4-b521351e5a8c.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..ed77d8c8bf12c33f02b6ec2e87b05c9aa2a18c76 GIT binary patch literal 12 TcmYc;N@ieSU}8|$Ro(~y51ImF literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/.part-00013-def8488a-2bf2-4623-bfba-f238deee6371.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/.part-00013-def8488a-2bf2-4623-bfba-f238deee6371.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..bc8291cd20f12c8746a34ebc38a2677570b012f8 GIT binary patch literal 12 TcmYc;N@ieSU}9+45%dNC6F390 literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/.part-00015-7ca44de2-8b15-465d-86f5-5de21ed67eee.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/.part-00015-7ca44de2-8b15-465d-86f5-5de21ed67eee.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..15b7ddafc70b0f24ab1d5101a23edd6abe4fbd48 GIT binary patch literal 12 TcmYc;N@ieSU}DI+;jsDX~mr zkMfcdi=k{YgATC(#P#drWU1n(hs1l-b43Zz;u$eX-~%I;ZP^vWrbVl5K*`#&lrX|> z*sx+;*F1e)YChme8fRY0>wFsM&`0MhGEoO)oSs1EGh$@chI{#b92kXt|ego#s&hC63JG*>NQ#4)SI@L)Qguc`5 sYT595jlPtZz1&Q@XC#Z`CZ@#tXa^07iJ(AI%JAO#lD@ literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/part-00011-ab578c13-0186-4f09-82e4-b521351e5a8c.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/part-00011-ab578c13-0186-4f09-82e4-b521351e5a8c.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..4ca9f6bb31a02ab4749fadaa8f6d430e7eba66c3 GIT binary patch literal 478 zcmZWmO-sW-5S^~I=1>HoyCjf9Sg^DZmp0pPMDQj(l;WWm5!oi&8ce^ERFTq)e?SEP zhd;-^<5X&Ma1YD8_h#ljnBnzpodUI~L*GBXzlt^`2n*BzIBgIBILe*izkAjhu}tHD zvXT;up=>jQ4zK{k?oBaSs`%+4@gMa}Q35o7PD~O8z=&mAcG<9L-f9|9uy!mZjIcf% zl#R=5Z&op=9E>v-Eio(F0kF>;EwK^30^CB%1{;(xk!zZZcq-IVVG8jZNIdH}2^Vwo zSPCg>U{?QiB&$eZ_SIUaB3`_RnNC%pQe}=dJbhhi-s4Fcul(M?qU`@`+QDa)IH%jmGL@^0;k>9 rvgUVcT`4;)?KeGN`V-aiWn(MpgZG owWCzKY5a!gD}St;zN(K}W~`J~Q=aE{>SgXYc!AFxzyL4%3(vP@#Q*>R literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/part-00015-7ca44de2-8b15-465d-86f5-5de21ed67eee.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table/date=2021-08-15/part-00015-7ca44de2-8b15-465d-86f5-5de21ed67eee.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c2cae0218154f75adff61abc6eb26a95a2c72c89 GIT binary patch literal 478 zcmZWmO-sW-5S^~IM392eT@uJ4ELd8IOPlRCB6yP?O7YN(h-{N>4W?g7sz~X@i+{jB z=Wp-_cy%hZIk<;q-g`6i9?bCiwoZZC)S+LW@83n65`+b602CVp0FH7e`0t)|Ml91f zpsb|CVkq0ppaU!bv3qlxELHsUkob>!rYHfLKPM&$17O6mExT;kG;cKxC|Em|5=K~` z4a&x4wl}L7R1U@&i+H zc`St#H889HI+9f+F#Bq)QxPv-#7w6uP^mIU8=k%{HSh5xjn{t4T|NnQ6ePTNx8ZX& z+eu-7^u8@x%%_Ov>nKv=2vr=@F8^D|dzxq children, EvalContext ctx) { + var leftChild = children.get(0); + var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); + var leftVal = leftChild.evalExpectValueAndType(ctx).getLeft(); + + var rightChild = children.get(1); + var rightType = rightChild.evalExpectValueAndType(ctx).getRight(); + var rightVal = rightChild.evalExpectValueAndType(ctx).getLeft(); + // If the types don't match, it implies a malformed predicate tree. + // We simply throw an exception, which will cause filtering to be skipped. + if (!Objects.equals(leftType, rightType)) { + throw new IllegalArgumentException( + "Type mismatch: " + leftType + " vs " + rightType + " for " + + leftChild + " and " + rightChild + ); + } + + // We throw an exception for nulls, which will skip filtering. + if (leftVal == null || rightVal == null) { + throw new IllegalArgumentException( + "Comparison with null is not supported: " + leftChild + " and " + rightChild + ); + } + + switch (leftType) { + case OpDataTypes.BoolType: return Boolean.valueOf(leftVal) == Boolean.valueOf(rightVal); + case OpDataTypes.IntType: return Integer.parseInt(leftVal) == Integer.parseInt(rightVal); + case OpDataTypes.LongType: return Long.parseLong(leftVal) == Long.parseLong(rightVal); + case OpDataTypes.StringType: return leftVal.equals(rightVal); + case OpDataTypes.DateType: + return java.sql.Date.valueOf(leftVal).equals(java.sql.Date.valueOf(rightVal)); + default: + throw new IllegalArgumentException("Unsupported type: " + leftType) ; + } + } + + static Boolean lessThan(List children, EvalContext ctx) { + var leftChild = children.get(0); + var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); + var leftVal = leftChild.evalExpectValueAndType(ctx).getLeft(); + + var rightChild = children.get(1); + var rightType = rightChild.evalExpectValueAndType(ctx).getRight(); + var rightVal = rightChild.evalExpectValueAndType(ctx).getLeft(); + // If the types don't match, it implies a malformed predicate tree. + // We simply throw an exception, which will cause filtering to be skipped. + if (!Objects.equals(leftType, rightType)) { + throw new IllegalArgumentException( + "Type mismatch: " + leftType + " vs " + rightType + " for " + + leftChild + " and " + rightChild + ); + } + + // We throw an exception for nulls, which will skip filtering. + if (leftVal == null || rightVal == null) { + throw new IllegalArgumentException( + "Comparison with null is not supported: " + leftChild + " and " + rightChild + ); + } + + switch (leftType) { +// case OpDataTypes.BoolType: return Boolean.valueOf(leftVal) == Boolean.valueOf(rightVal); + case OpDataTypes.IntType: return Integer.parseInt(leftVal) < Integer.parseInt(rightVal); + case OpDataTypes.LongType: return Long.parseLong(leftVal) < Long.parseLong(rightVal); + case OpDataTypes.StringType: return leftVal.compareTo(rightVal) < 0; + case OpDataTypes.DateType: + return java.sql.Date.valueOf(leftVal).equals(java.sql.Date.valueOf(rightVal)); + default: + throw new IllegalArgumentException("Unsupported type: " + leftType) ; + } + } + // Validates that the specified value is in the correct format. + // Throws an exception otherwise. + public static void validateValue(String value, String valueType) { + try { + switch (valueType) { + case OpDataTypes.BoolType: Boolean.parseBoolean(value); + case OpDataTypes.IntType: Integer.parseInt(value); + case OpDataTypes.LongType: Long.parseLong(value); + case OpDataTypes.DateType: java.sql.Date.valueOf(value); + case OpDataTypes.FloatType: Float.parseFloat(value); + case OpDataTypes.DoubleType: Double.parseDouble(value); + // TODO check for non deprecated + case OpDataTypes.TimestampType: Timestamp.parse(value); + default: + throw new IllegalArgumentException("Unsupported type: " + valueType); + } + } catch (Exception e) { + throw new IllegalArgumentException( + "Error validating " + value + " for type " + valueType + ": " + e + ); + } + } + } \ No newline at end of file diff --git a/server/src/main/java/io/whitefox/core/JsonPredicates.java b/server/src/main/java/io/whitefox/core/JsonPredicates.java new file mode 100644 index 000000000..588c8fa2f --- /dev/null +++ b/server/src/main/java/io/whitefox/core/JsonPredicates.java @@ -0,0 +1,385 @@ +package io.whitefox.core; + +import org.apache.commons.lang3.tuple.Pair; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + + +class EvalContext { + + public EvalContext(Map partitionValues, Map> statsValues) { + this.partitionValues = partitionValues; + this.statsValues = statsValues; + } + + Map partitionValues; + Map> statsValues; +} + +class OpDataTypes { + static final String BoolType = "bool"; + static final String IntType = "int"; + static final String LongType = "long"; + static final String StringType = "string"; + static final String DateType = "date"; + static final String FloatType = "float"; + static final String DoubleType = "double"; + static final String TimestampType = "timestamp"; + + static List supportedTypes = List.of(BoolType, IntType, LongType, StringType, DateType); + static List supportedTypesV2 = Stream.concat(supportedTypes.stream(), Stream.of(FloatType, DoubleType, TimestampType)).collect(Collectors.toList()); + + + static Boolean isSupportedType(String valueType, Boolean forV2) { + if (forV2) { + return OpDataTypes.supportedTypesV2.contains(valueType); + } else { + return OpDataTypes.supportedTypes.contains(valueType); + } + } +} + +interface BaseOp { + void validate(); + + Object eval(EvalContext ctx); + + default Boolean evalExpectBoolean(EvalContext ctx) { + return (Boolean) eval(ctx); + } + + List getAllChildren(); + + default Boolean treeDepthExceeds(Integer depth) { + if (depth <= 0) { + return true; + } else { + return getAllChildren().stream().anyMatch(c -> c.treeDepthExceeds(depth - 1)); + } + } +} + + +// Represents a leaf operation. +abstract class LeafOp implements BaseOp { + + abstract Boolean isNull(EvalContext ctx); + + Pair evalExpectValueAndType(EvalContext ctx) { + return (Pair) eval(ctx); + } + + @Override + public List getAllChildren() { + return List.of(); + } + + abstract String getOpValueType(); +} + +// Represents a non-leaf operation. +abstract class NonLeafOp implements BaseOp { + List children; + + + public List getAllChildren() { + return (List) List.copyOf(children).stream().map(c -> (BaseOp) c); + } +} + +// Represents a unary operation. +interface UnaryOp { + // Validates number of children to be 1. + default void validateChildren(List children) { + if (children.size() != 1) + throw new IllegalArgumentException( + this + " : expected 1 but found " + children.size() + " children" + ); + children.get(0).validate(); + } +} + +interface BinaryOp { + // Validates number of children to be 2. + default void validateChildren(List children) { + if (children.size() != 2) + throw new IllegalArgumentException( + this + " : expected 2 but found " + children.size() + " children" + ); + children.forEach(BaseOp::validate); + var child1 = children.get(0); + var child2 = children.get(1); + if (child1 instanceof LeafOp && child2 instanceof LeafOp) { + var leftType = ((LeafOp) child1).getOpValueType(); + var rightType = ((LeafOp) child2).getOpValueType(); + if (!Objects.equals(leftType, rightType)) { + throw new IllegalArgumentException( + "Type mismatch: " + leftType + " vs " + rightType + " for " + + child1 + " and " + child2 + ); + } + } + } +} + +interface NaryOp { + // Validates number of children to be at least 2. + default void validateChildren(List children) { + if (children.size() < 2) { + throw new IllegalArgumentException( + this + " : expected at least 2 but found " + children.size() + " children" + ); + } + children.forEach(BaseOp::validate); + } +} + +class ColumnOp extends LeafOp { + + String name; + String valueType; + + public ColumnOp(String name, String valueType) { + this.name = name; + this.valueType = valueType; + } + + // Determine if the column value is null. + @Override + public Boolean isNull(EvalContext ctx) { + return resolve(ctx) == null; + } + + @Override + public Boolean evalExpectBoolean(EvalContext ctx) { + if (!Objects.equals(valueType, OpDataTypes.BoolType)) { + throw new IllegalArgumentException( + "Unsupported type for boolean evaluation: " + valueType + ); + } + return Boolean.valueOf(resolve(ctx)); + } + + @Override + public String getOpValueType() { + return null; + } + + @Override + public Object eval(EvalContext ctx) { + return Pair.of(resolve(ctx), valueType); + } + + public void validate() { + if (name == null) { + throw new IllegalArgumentException("Name must be specified: " + this); + } + if (!OpDataTypes.isSupportedType(valueType, false)) { + throw new IllegalArgumentException("Unsupported type: " + valueType); + } + } + + private String resolve(EvalContext ctx) { + return ctx.partitionValues.getOrDefault(name, null); + } +} + +class LiteralOp extends LeafOp { + + String value; + String valueType; + + @Override + public void validate() { + if (value == null) { + throw new IllegalArgumentException("Value must be specified: " + this); + } + if (!OpDataTypes.isSupportedType(valueType, false)) { + throw new IllegalArgumentException("Unsupported type: " + valueType); + } + EvalHelper.validateValue(value, valueType); + } + + @Override + public Object eval(EvalContext ctx) { + return Pair.of(value, valueType); + } + + public LiteralOp(String value, String valueType) { + this.value = value; + this.valueType = valueType; + } + + @Override + public Boolean isNull(EvalContext ctx) { + return false; + } + + + @Override + public String getOpValueType() { + return valueType; + } +} + +class IsNullOp extends NonLeafOp implements UnaryOp { + + public IsNullOp(List children) { + this.children = children; + } + + @Override + public void validate() { + + } + + @Override + public Object eval(EvalContext ctx) { + return children.get(0).isNull(ctx); + } + + +} + +class EqualOp extends NonLeafOp implements BinaryOp { + List children; + + public EqualOp(List children) { + this.children = children; + } + + @Override + public void validate() { + validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); + } + + @Override + public Object eval(EvalContext ctx) { + return EvalHelper.equal(children, ctx); + } +} + +class LessThanOp extends NonLeafOp implements BinaryOp { + List children; + + public LessThanOp(List children) { + this.children = children; + } + + @Override + public void validate() { + validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); + } + + @Override + public Object eval(EvalContext ctx) { + return EvalHelper.lessThan(children, ctx); + } +} + +class LessThanOrEqualOp extends NonLeafOp implements BinaryOp { + + public LessThanOrEqualOp(List children) { + this.children = children; + } + + @Override + public void validate() { + validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); + } + + @Override + public Object eval(EvalContext ctx) { + return EvalHelper.lessThan(children, ctx) || EvalHelper.equal(children, ctx); + } +} + + +class GreaterThanOp extends NonLeafOp implements BinaryOp { + + public GreaterThanOp(List children) { + this.children = children; + } + + @Override + public void validate() { + validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); + } + + @Override + public Object eval(EvalContext ctx) { + return !EvalHelper.lessThan(children, ctx) && !EvalHelper.equal(children, ctx); + } +} + +class GreaterThanOrEqualOp extends NonLeafOp implements BinaryOp { + + public GreaterThanOrEqualOp(List children) { + this.children = children; + } + + @Override + public void validate() { + validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); + } + + @Override + public Object eval(EvalContext ctx) { + return !EvalHelper.lessThan(children, ctx); + } +} + +class AndOp extends NonLeafOp implements BinaryOp { + + public AndOp(List children) { + this.children = children; + } + + @Override + public void validate() { + validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); + } + + @Override + public Object eval(EvalContext ctx) { + return children.stream().allMatch(c -> c.evalExpectBoolean(ctx)); + } +} + +class OrOp extends NonLeafOp implements BinaryOp { + + public OrOp(List children) { + this.children = children; + } + + @Override + public void validate() { + validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); + } + + @Override + public Object eval(EvalContext ctx) { + return children.stream().anyMatch(c -> c.evalExpectBoolean(ctx)); + } +} + +class NotOp extends NonLeafOp implements UnaryOp { + + public NotOp(List children) { + this.children = children; + } + + @Override + public void validate() { + validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); + } + + @Override + public Object eval(EvalContext ctx) { + return !children.get(0).evalExpectBoolean(ctx); + } +} \ No newline at end of file diff --git a/server/src/main/java/io/whitefox/core/PartitionFilterUtils.java b/server/src/main/java/io/whitefox/core/PartitionFilterUtils.java new file mode 100644 index 000000000..8f4cb3f0d --- /dev/null +++ b/server/src/main/java/io/whitefox/core/PartitionFilterUtils.java @@ -0,0 +1,5 @@ +package io.whitefox.core; + +public class PartitionFilterUtils { + +} From 1e085cdbb0c8943df30b05228215f3b9be777d94 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Sat, 18 Nov 2023 01:50:19 +0100 Subject: [PATCH 02/30] wip - working parsing --- .../predicates/PredicateParsingTest.java | 55 +++ .../core/services/DeltaSharedTable.java | 21 +- .../core/types/predicates/BaseOp.java | 109 +++++ .../predicates/DataTypeDeserializer.java | 46 +++ .../core/types/predicates/EvalContext.java | 17 + .../core/types/predicates/EvalHelper.java | 105 +++++ .../core/types/predicates/LeafOp.java | 142 +++++++ .../types/predicates/LeafOpDeserializer.java | 39 ++ .../core/types/predicates/NonLeafOp.java | 259 ++++++++++++ .../core/services/DeltaSharedTableTest.java | 7 +- .../java/io/whitefox/core/EvalHelper.java | 103 ----- .../java/io/whitefox/core/JsonPredicates.java | 385 ------------------ .../whitefox/core/PartitionFilterUtils.java | 5 - .../core/types/predicates/BaseOp.java | 109 +++++ .../predicates/DataTypeDeserializer.java | 46 +++ .../core/types/predicates/EvalContext.java | 17 + .../core/types/predicates/EvalHelper.java | 105 +++++ .../core/types/predicates/LeafOp.java | 142 +++++++ .../types/predicates/LeafOpDeserializer.java | 39 ++ .../core/types/predicates/NonLeafOp.java | 259 ++++++++++++ 20 files changed, 1512 insertions(+), 498 deletions(-) create mode 100644 server/app/src/test/java/io/whitefox/api/core/types/predicates/PredicateParsingTest.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/EvalContext.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java delete mode 100644 server/src/main/java/io/whitefox/core/EvalHelper.java delete mode 100644 server/src/main/java/io/whitefox/core/JsonPredicates.java delete mode 100644 server/src/main/java/io/whitefox/core/PartitionFilterUtils.java create mode 100644 server/src/main/java/io/whitefox/core/types/predicates/BaseOp.java create mode 100644 server/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java create mode 100644 server/src/main/java/io/whitefox/core/types/predicates/EvalContext.java create mode 100644 server/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java create mode 100644 server/src/main/java/io/whitefox/core/types/predicates/LeafOp.java create mode 100644 server/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java create mode 100644 server/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java diff --git a/server/app/src/test/java/io/whitefox/api/core/types/predicates/PredicateParsingTest.java b/server/app/src/test/java/io/whitefox/api/core/types/predicates/PredicateParsingTest.java new file mode 100644 index 000000000..d741eff0b --- /dev/null +++ b/server/app/src/test/java/io/whitefox/api/core/types/predicates/PredicateParsingTest.java @@ -0,0 +1,55 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.core.JsonProcessingException; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static io.whitefox.core.services.DeltaSharedTable.parsePredicate; + +public class PredicateParsingTest { + + @Test + void testParsingOfEqual() throws JsonProcessingException { + var predicate = "{\n" + + " \"op\": \"equal\",\n" + + " \"children\": [\n" + + " {\"op\": \"column\", \"name\":\"hireDate\", \"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + + " ]\n" + + "}"; + var op = parsePredicate(predicate); + op.validate(); + EvalContext evctx = new EvalContext(Map.of("date", "20100312"), Map.of()); + op.eval(evctx); + } + + + @Test + void testParsingOfNested() throws JsonProcessingException { + var predicate = "{\n" + + " \"op\":\"and\",\n" + + " \"children\":[\n" + + " {\n" + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"hireDate\",\"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"op\":\"lessThan\",\"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + + " {\"op\":\"literal\",\"value\":\"25\",\"valueType\":\"int\"}\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + var op = parsePredicate(predicate); + op.validate(); + EvalContext evctx = new EvalContext(Map.of("hireDate", "20100312"), Map.of()); + op.eval(evctx); + } + + +} diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index a32c77891..f8ec7726a 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -1,5 +1,7 @@ package io.whitefox.core.services; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.standalone.DeltaLog; import io.delta.standalone.Snapshot; import io.delta.standalone.actions.AddFile; @@ -10,6 +12,8 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import io.whitefox.core.types.predicates.BaseOp; +import org.apache.hadoop.conf.Configuration; public class DeltaSharedTable implements InternalSharedTable { @@ -80,24 +84,37 @@ public Optional getTableVersion(Optional startingTimestamp) { return getSnapshot(startingTimestamp).map(Snapshot::getVersion); } + public static BaseOp parsePredicate(String predicate) throws JsonProcessingException { + ObjectMapper mapper = new ObjectMapper(); + try { + return mapper.readValue(predicate, BaseOp.class); + } catch (JsonProcessingException e) { + System.out.println("cant parse predicate"); + throw e; + } + } + public boolean filterFilesBasedOnPredicates(List predicates, AddFile f) { var partitionValues = f.getPartitionValues(); - predicates.forEach(p -> { + predicates.forEach(p -> {}); - }); return true; } public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { + List predicates; Snapshot snapshot; if (readTableRequest instanceof ReadTableRequest.ReadTableCurrentVersion) { snapshot = deltaLog.snapshot(); + predicates = ((ReadTableRequest.ReadTableCurrentVersion) readTableRequest).predicateHints(); } else if (readTableRequest instanceof ReadTableRequest.ReadTableAsOfTimestamp) { snapshot = deltaLog.getSnapshotForTimestampAsOf( ((ReadTableRequest.ReadTableAsOfTimestamp) readTableRequest).timestamp()); + predicates = ((ReadTableRequest.ReadTableAsOfTimestamp) readTableRequest).predicateHints(); } else if (readTableRequest instanceof ReadTableRequest.ReadTableVersion) { snapshot = deltaLog.getSnapshotForVersionAsOf( ((ReadTableRequest.ReadTableVersion) readTableRequest).version()); + predicates = ((ReadTableRequest.ReadTableVersion) readTableRequest).predicateHints(); } else { throw new IllegalArgumentException("Unknown ReadTableRequest type: " + readTableRequest); } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java new file mode 100644 index 000000000..3ccdcfab1 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java @@ -0,0 +1,109 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.whitefox.core.types.*; + +import java.util.List; +import java.util.Objects; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "op") +@JsonSubTypes({ + @JsonSubTypes.Type(value = LeafOp.class, names = {"column", "literal"}), +// @JsonSubTypes.Type(value = EqualOp.class, names = {"equal"/*, "not", "or", "and", "lessThan", "lessThanOrEqual", "greaterThan", "greaterThanOrEqualOp" */}), +// @JsonSubTypes.Type(value = ColumnOp.class, name = "column"), +// @JsonSubTypes.Type(value = LiteralOp.class, name = "literal"), + @JsonSubTypes.Type(value = EqualOp.class, name = "equal"), + @JsonSubTypes.Type(value = NotOp.class, name = "not"), + @JsonSubTypes.Type(value = OrOp.class, name = "or"), + @JsonSubTypes.Type(value = AndOp.class, name = "and"), + @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), + @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), + @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), + @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqualOp") +}) +public interface BaseOp { + + void validate(); + + default Boolean isSupportedType(DataType valueType, Boolean forV2) { + if (forV2) { + return (valueType instanceof BooleanType + || valueType instanceof IntegerType + || valueType instanceof StringType + || valueType instanceof DateType + || valueType instanceof LongType + || valueType instanceof TimestampType + || valueType instanceof FloatType + || valueType instanceof DoubleType); + } else { + return (valueType instanceof BooleanType + || valueType instanceof IntegerType + || valueType instanceof StringType + || valueType instanceof DateType + || valueType instanceof LongType); + } + } + + Object eval(EvalContext ctx); + + default Boolean evalExpectBoolean(EvalContext ctx) { + return (Boolean) eval(ctx); + } + + List getAllChildren(); + + default Boolean treeDepthExceeds(Integer depth) { + if (depth <= 0) { + return true; + } else { + return getAllChildren().stream().anyMatch(c -> c.treeDepthExceeds(depth - 1)); + } + } +} + + +// Represents a unary operation. +interface UnaryOp { + // Validates number of children to be 1. + default void validateChildren(List children) { + if (children.size() != 1) + throw new IllegalArgumentException( + this + " : expected 1 but found " + children.size() + " children"); + children.get(0).validate(); + } +} + +interface BinaryOp { + // Validates number of children to be 2. + default void validateChildren(List children) { + if (children.size() != 2) + throw new IllegalArgumentException( + this + " : expected 2 but found " + children.size() + " children"); + children.forEach(BaseOp::validate); + var child1 = children.get(0); + var child2 = children.get(1); + if (child1 instanceof LeafOp && child2 instanceof LeafOp) { + var leftType = ((LeafOp) child1).getOpValueType(); + var rightType = ((LeafOp) child2).getOpValueType(); + if (!Objects.equals(leftType, rightType)) { + throw new IllegalArgumentException("Type mismatch: " + leftType + " vs " + rightType + + " for " + child1 + " and " + child2); + } + } + } +} + +interface NaryOp { + // Validates number of children to be at least 2. + default void validateChildren(List children) { + if (children.size() < 2) { + throw new IllegalArgumentException( + this + " : expected at least 2 but found " + children.size() + " children"); + } + children.forEach(BaseOp::validate); + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java b/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java new file mode 100644 index 000000000..7f560908c --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java @@ -0,0 +1,46 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import io.whitefox.core.types.*; + +import java.io.IOException; + +public class DataTypeDeserializer extends StdDeserializer { + + // needed for jackson + public DataTypeDeserializer() { + this(null); + } + + public DataTypeDeserializer(Class vc) { + super(vc); + } + + @Override + public DataType deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + String valueType = node.asText(); + + switch (valueType) { + case "date": + return DateType.DATE; + case "int": + return IntegerType.INTEGER; + case "double": + return DoubleType.DOUBLE; + case "float": + return FloatType.FLOAT; + case "string": + return StringType.STRING; + case "timestamp": + return TimestampType.TIMESTAMP; + case "long": + return LongType.LONG; + default: + throw new IOException("Unknown type passed inside a json predicate: " + valueType); + } + } +} \ No newline at end of file diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalContext.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalContext.java new file mode 100644 index 000000000..3320d4eee --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalContext.java @@ -0,0 +1,17 @@ +package io.whitefox.core.types.predicates; + +import org.apache.commons.lang3.tuple.Pair; + +import java.util.Map; + +class EvalContext { + + public EvalContext( + Map partitionValues, Map> statsValues) { + this.partitionValues = partitionValues; + this.statsValues = statsValues; + } + + Map partitionValues; + Map> statsValues; +} \ No newline at end of file diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java new file mode 100644 index 000000000..0da4645af --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -0,0 +1,105 @@ +package io.whitefox.core.types.predicates; + +import io.whitefox.core.types.*; +import java.sql.Date; +import java.sql.Timestamp; +import java.util.List; +import java.util.Objects; + +import io.whitefox.core.types.predicates.LeafOp; +import org.apache.commons.lang3.tuple.Pair; + +public class EvalHelper { + + static Pair, Pair> validateAndGetTypeAndValue( + List children, EvalContext ctx) { + var leftChild = children.get(0); + var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); + var leftVal = leftChild.evalExpectValueAndType(ctx).getLeft(); + + var rightChild = children.get(1); + var rightType = rightChild.evalExpectValueAndType(ctx).getRight(); + var rightVal = rightChild.evalExpectValueAndType(ctx).getLeft(); + // If the types don't match, it implies a malformed predicate tree. + // We simply throw an exception, which will cause filtering to be skipped. + if (!Objects.equals(leftType, rightType)) { + throw new IllegalArgumentException("Type mismatch: " + leftType + " vs " + rightType + " for " + + leftChild + " and " + rightChild); + } + + // We throw an exception for nulls, which will skip filtering. + if (leftVal == null || rightVal == null) { + throw new IllegalArgumentException( + "Comparison with null is not supported: " + leftChild + " and " + rightChild); + } + return Pair.of(Pair.of(leftType, leftVal), Pair.of(rightType, rightVal)); + } + + // Implements "equal" between two leaf operations. + static Boolean equal(List children, EvalContext ctx) { + var typesAndValues = validateAndGetTypeAndValue(children, ctx); + var leftType = typesAndValues.getLeft().getLeft(); + var leftVal = typesAndValues.getLeft().getRight(); + var rightVal = typesAndValues.getRight().getRight(); + + if (BooleanType.BOOLEAN.equals(leftType)) { + return Boolean.valueOf(leftVal) == Boolean.valueOf(rightVal); + } else if (IntegerType.INTEGER.equals(leftType)) { + return Integer.parseInt(leftVal) == Integer.parseInt(rightVal); + } else if (LongType.LONG.equals(leftType)) { + return Long.parseLong(leftVal) == Long.parseLong(rightVal); + } else if (StringType.STRING.equals(leftType)) { + return leftVal.equals(rightVal); + } else if (DateType.DATE.equals(leftType)) { + return Date.valueOf(leftVal).equals(Date.valueOf(rightVal)); + } + throw new IllegalArgumentException("Unsupported type: " + leftType); + } + + // TODO: supported expressions; ie. check if column + constant + // TODO: handle column comparisons with literals + static Boolean lessThan(List children, EvalContext ctx) { + var typesAndValues = validateAndGetTypeAndValue(children, ctx); + var leftType = typesAndValues.getLeft().getLeft(); + var leftVal = typesAndValues.getLeft().getRight(); + var rightVal = typesAndValues.getRight().getRight(); + + if (IntegerType.INTEGER.equals(leftType)) { + return Integer.parseInt(leftVal) < Integer.parseInt(rightVal); + } else if (LongType.LONG.equals(leftType)) { + return Long.parseLong(leftVal) < Long.parseLong(rightVal); + } else if (StringType.STRING.equals(leftType)) { + return leftVal.compareTo(rightVal) < 0; + } else if (DateType.DATE.equals(leftType)) { + return Date.valueOf(leftVal).before(Date.valueOf(rightVal)); + } + throw new IllegalArgumentException("Unsupported type: " + leftType); + } + // Validates that the specified value is in the correct format. + // Throws an exception otherwise. + public static void validateValue(String value, DataType valueType) { + try { + if (BooleanType.BOOLEAN.equals(valueType)) { + Boolean.parseBoolean(value); + } else if (IntegerType.INTEGER.equals(valueType)) { + Integer.parseInt(value); + } else if (LongType.LONG.equals(valueType)) { + Long.parseLong(value); + } else if (DateType.DATE.equals(valueType)) { + Date.valueOf(value); + } else if (FloatType.FLOAT.equals(valueType)) { + Float.parseFloat(value); + } else if (DoubleType.DOUBLE.equals(valueType)) { + Double.parseDouble(value); + // TODO check for non deprecated + } else if (TimestampType.TIMESTAMP.equals(valueType)) { + Timestamp.valueOf(value); + } else { + throw new IllegalArgumentException("Unsupported type: " + valueType); + } + } catch (Exception e) { + throw new IllegalArgumentException( + "Error validating " + value + " for type " + valueType + ": " + e); + } + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java new file mode 100644 index 000000000..64a4c8cd3 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -0,0 +1,142 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import io.whitefox.core.types.BooleanType; +import io.whitefox.core.types.DataType; +import org.apache.commons.lang3.tuple.Pair; + +//import java.beans.ConstructorProperties; +import java.util.List; +import java.util.Objects; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "op" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = ColumnOp.class, name = "column"), + @JsonSubTypes.Type(value = LiteralOp.class, name = "literal")} +) +//@JsonDeserialize(using = LeafOpDeserializer.class) +public abstract class LeafOp implements BaseOp { + + abstract Boolean isNull(EvalContext ctx); + + @JsonDeserialize(using = DataTypeDeserializer.class) + @JsonProperty("valueType") + DataType valueType; + + Pair evalExpectValueAndType(EvalContext ctx) { + return (Pair) eval(ctx); + } + + @Override + public List getAllChildren() { + return List.of(); + } + + abstract DataType getOpValueType(); +} + + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "column") +class ColumnOp extends LeafOp { + + @JsonProperty("name") + String name; + + public ColumnOp(){ + super(); + } + + public ColumnOp(String name, DataType valueType) { + this.name = name; + this.valueType = valueType; + } + + // Determine if the column value is null. + @Override + public Boolean isNull(EvalContext ctx) { + return resolve(ctx) == null; + } + + @Override + public Boolean evalExpectBoolean(EvalContext ctx) { + if (!Objects.equals(valueType, BooleanType.BOOLEAN)) { + throw new IllegalArgumentException("Unsupported type for boolean evaluation: " + valueType); + } + return Boolean.valueOf(resolve(ctx)); + } + + @Override + public DataType getOpValueType() { + return valueType; + } + + @Override + public Object eval(EvalContext ctx) { + return Pair.of(resolve(ctx), valueType); + } + + public void validate() { + if (name == null) { + throw new IllegalArgumentException("Name must be specified: " + this); + } + if (!this.isSupportedType(valueType, false)) { + throw new IllegalArgumentException("Unsupported type: " + valueType); + } + } + + private String resolve(EvalContext ctx) { + return ctx.partitionValues.getOrDefault(name, null); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "literal") +class LiteralOp extends LeafOp { + @JsonProperty("value") + String value; + + @Override + public void validate() { + if (value == null) { + throw new IllegalArgumentException("Value must be specified: " + this); + } + if (!isSupportedType(valueType, false)) { + throw new IllegalArgumentException("Unsupported type: " + valueType); + } + EvalHelper.validateValue(value, valueType); + } + + @Override + public Object eval(EvalContext ctx) { + return Pair.of(value, valueType); + } + + public LiteralOp() { + super(); + } + + public LiteralOp(String value, DataType valueType) { + this.value = value; + this.valueType = valueType; + } + + @Override + public Boolean isNull(EvalContext ctx) { + return false; + } + + @Override + public DataType getOpValueType() { + return valueType; + } +} \ No newline at end of file diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java new file mode 100644 index 000000000..4e4824e00 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java @@ -0,0 +1,39 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import io.whitefox.core.types.DateType; + +import java.io.IOException; + +public class LeafOpDeserializer extends StdDeserializer { + public LeafOpDeserializer() { + this(null); + } + + public LeafOpDeserializer(Class vc) { + super(vc); + } + + @Override + public LeafOp deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + String valueType = node.get("valueType").asText(); + String op = node.get("op").asText(); + + // For example: + switch (op) { + case "column": + String name = node.get("name").asText(); +// switch (valueType) + return new ColumnOp(name, DateType.DATE); + case "literal": + String value = node.get("value").asText(); + return new LiteralOp(value, DateType.DATE); + default: + throw new IOException("type not known"); + } + } +} \ No newline at end of file diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java new file mode 100644 index 000000000..ac47252d3 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java @@ -0,0 +1,259 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.util.List; +import java.util.stream.Collectors; + +// Represents a non-leaf operation. +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "op" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = EqualOp.class, name = "equal"), + @JsonSubTypes.Type(value = NotOp.class, name = "not"), + @JsonSubTypes.Type(value = OrOp.class, name = "or"), + @JsonSubTypes.Type(value = IsNullOp.class, name = "null"), + @JsonSubTypes.Type(value = AndOp.class, name = "and"), + @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), + @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), + @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), + @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqualOp") +}) +public abstract class NonLeafOp implements BaseOp { + + @JsonProperty("children") + List children; + + public List getAllChildren() { + // TODO flat map every child + return List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList()); + } +} + +class IsNullOp extends NonLeafOp implements UnaryOp { + + @JsonProperty("children") + List children; + + public IsNullOp(List children) { + this.children = children; + } + + public IsNullOp() { + super(); + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return ((LeafOp) children.get(0)).isNull(ctx); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "equal") +class EqualOp extends NonLeafOp implements BinaryOp { + @JsonProperty("children") + List children; + + public EqualOp() { + super(); + } + + public EqualOp(List children) { + this.children = children; + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return EvalHelper.equal(children, ctx); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "lessThan") +class LessThanOp extends NonLeafOp implements BinaryOp { + @JsonProperty("children") + List children; + + public LessThanOp(List children) { + this.children = children; + } + + public LessThanOp() { + super(); + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return EvalHelper.lessThan(children, ctx); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "lessThanOrEqual") +class LessThanOrEqualOp extends NonLeafOp implements BinaryOp { + + @JsonProperty("children") + List children; + + public LessThanOrEqualOp(List children) { + this.children = children; + } + + public LessThanOrEqualOp() { + super(); + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return EvalHelper.lessThan(children, ctx) || EvalHelper.equal(children, ctx); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "greaterThan") +class GreaterThanOp extends NonLeafOp implements BinaryOp { + + @JsonProperty("children") + List children; + + public GreaterThanOp() { + super(); + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return !EvalHelper.lessThan(children, ctx) && !EvalHelper.equal(children, ctx); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "greaterThanOrEqual") +class GreaterThanOrEqualOp extends NonLeafOp implements BinaryOp { + + @JsonProperty("children") + List children; + + public GreaterThanOrEqualOp() { + super(); + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return !EvalHelper.lessThan(children, ctx); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "and") +class AndOp extends NonLeafOp implements BinaryOp { + + public AndOp(List children) { + this.children = children; + } + + public AndOp() { + super(); + } + + @Override + public void validate() { + validateChildren(children); + } + + @Override + public Object eval(EvalContext ctx) { + return children.stream().allMatch(c -> c.evalExpectBoolean(ctx)); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "or") +class OrOp extends NonLeafOp implements BinaryOp { + + public OrOp(List children) { + this.children = children; + } + + public OrOp() { + super(); + } + + @Override + public void validate() { + validateChildren(children); + } + + @Override + public Object eval(EvalContext ctx) { + return children.stream().anyMatch(c -> c.evalExpectBoolean(ctx)); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "not") +class NotOp extends NonLeafOp implements UnaryOp { + + @JsonProperty("children") + List children; + + public NotOp(List children) { + this.children = children; + } + + public NotOp() { + super(); + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return !children.get(0).evalExpectBoolean(ctx); + } +} \ No newline at end of file diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index f03fe3147..f6a519e30 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -5,7 +5,6 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.wildfly.common.Assert.assertTrue; -import io.whitefox.core.Protocol; import io.whitefox.core.ReadTableRequest; import io.whitefox.core.SharedTable; import java.time.format.DateTimeParseException; @@ -79,9 +78,11 @@ void getTableVersionWithMalformedTimestamp() throws ExecutionException, Interrup @Test void queryTable() throws ExecutionException, InterruptedException { - var PTable = new SharedTable("partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); + var PTable = new SharedTable( + "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); - var request = new ReadTableRequest.ReadTableCurrentVersion(List.of("date = '2021-08-09'"), Optional.empty()); + var request = new ReadTableRequest.ReadTableCurrentVersion( + List.of("date = '2021-08-09'"), Optional.empty()); var response = DTable.queryTable(request); assertTrue(2 == 2); } diff --git a/server/src/main/java/io/whitefox/core/EvalHelper.java b/server/src/main/java/io/whitefox/core/EvalHelper.java deleted file mode 100644 index ec82d37e6..000000000 --- a/server/src/main/java/io/whitefox/core/EvalHelper.java +++ /dev/null @@ -1,103 +0,0 @@ -package io.whitefox.core; - -import java.sql.Timestamp; -import java.util.List; -import java.util.Objects; - -public class EvalHelper { - - // Implements "equal" between two leaf operations. - static Boolean equal(List children, EvalContext ctx) { - var leftChild = children.get(0); - var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); - var leftVal = leftChild.evalExpectValueAndType(ctx).getLeft(); - - var rightChild = children.get(1); - var rightType = rightChild.evalExpectValueAndType(ctx).getRight(); - var rightVal = rightChild.evalExpectValueAndType(ctx).getLeft(); - // If the types don't match, it implies a malformed predicate tree. - // We simply throw an exception, which will cause filtering to be skipped. - if (!Objects.equals(leftType, rightType)) { - throw new IllegalArgumentException( - "Type mismatch: " + leftType + " vs " + rightType + " for " + - leftChild + " and " + rightChild - ); - } - - // We throw an exception for nulls, which will skip filtering. - if (leftVal == null || rightVal == null) { - throw new IllegalArgumentException( - "Comparison with null is not supported: " + leftChild + " and " + rightChild - ); - } - - switch (leftType) { - case OpDataTypes.BoolType: return Boolean.valueOf(leftVal) == Boolean.valueOf(rightVal); - case OpDataTypes.IntType: return Integer.parseInt(leftVal) == Integer.parseInt(rightVal); - case OpDataTypes.LongType: return Long.parseLong(leftVal) == Long.parseLong(rightVal); - case OpDataTypes.StringType: return leftVal.equals(rightVal); - case OpDataTypes.DateType: - return java.sql.Date.valueOf(leftVal).equals(java.sql.Date.valueOf(rightVal)); - default: - throw new IllegalArgumentException("Unsupported type: " + leftType) ; - } - } - - static Boolean lessThan(List children, EvalContext ctx) { - var leftChild = children.get(0); - var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); - var leftVal = leftChild.evalExpectValueAndType(ctx).getLeft(); - - var rightChild = children.get(1); - var rightType = rightChild.evalExpectValueAndType(ctx).getRight(); - var rightVal = rightChild.evalExpectValueAndType(ctx).getLeft(); - // If the types don't match, it implies a malformed predicate tree. - // We simply throw an exception, which will cause filtering to be skipped. - if (!Objects.equals(leftType, rightType)) { - throw new IllegalArgumentException( - "Type mismatch: " + leftType + " vs " + rightType + " for " + - leftChild + " and " + rightChild - ); - } - - // We throw an exception for nulls, which will skip filtering. - if (leftVal == null || rightVal == null) { - throw new IllegalArgumentException( - "Comparison with null is not supported: " + leftChild + " and " + rightChild - ); - } - - switch (leftType) { -// case OpDataTypes.BoolType: return Boolean.valueOf(leftVal) == Boolean.valueOf(rightVal); - case OpDataTypes.IntType: return Integer.parseInt(leftVal) < Integer.parseInt(rightVal); - case OpDataTypes.LongType: return Long.parseLong(leftVal) < Long.parseLong(rightVal); - case OpDataTypes.StringType: return leftVal.compareTo(rightVal) < 0; - case OpDataTypes.DateType: - return java.sql.Date.valueOf(leftVal).equals(java.sql.Date.valueOf(rightVal)); - default: - throw new IllegalArgumentException("Unsupported type: " + leftType) ; - } - } - // Validates that the specified value is in the correct format. - // Throws an exception otherwise. - public static void validateValue(String value, String valueType) { - try { - switch (valueType) { - case OpDataTypes.BoolType: Boolean.parseBoolean(value); - case OpDataTypes.IntType: Integer.parseInt(value); - case OpDataTypes.LongType: Long.parseLong(value); - case OpDataTypes.DateType: java.sql.Date.valueOf(value); - case OpDataTypes.FloatType: Float.parseFloat(value); - case OpDataTypes.DoubleType: Double.parseDouble(value); - // TODO check for non deprecated - case OpDataTypes.TimestampType: Timestamp.parse(value); - default: - throw new IllegalArgumentException("Unsupported type: " + valueType); - } - } catch (Exception e) { - throw new IllegalArgumentException( - "Error validating " + value + " for type " + valueType + ": " + e - ); - } - } - } \ No newline at end of file diff --git a/server/src/main/java/io/whitefox/core/JsonPredicates.java b/server/src/main/java/io/whitefox/core/JsonPredicates.java deleted file mode 100644 index 588c8fa2f..000000000 --- a/server/src/main/java/io/whitefox/core/JsonPredicates.java +++ /dev/null @@ -1,385 +0,0 @@ -package io.whitefox.core; - -import org.apache.commons.lang3.tuple.Pair; - -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; - - -class EvalContext { - - public EvalContext(Map partitionValues, Map> statsValues) { - this.partitionValues = partitionValues; - this.statsValues = statsValues; - } - - Map partitionValues; - Map> statsValues; -} - -class OpDataTypes { - static final String BoolType = "bool"; - static final String IntType = "int"; - static final String LongType = "long"; - static final String StringType = "string"; - static final String DateType = "date"; - static final String FloatType = "float"; - static final String DoubleType = "double"; - static final String TimestampType = "timestamp"; - - static List supportedTypes = List.of(BoolType, IntType, LongType, StringType, DateType); - static List supportedTypesV2 = Stream.concat(supportedTypes.stream(), Stream.of(FloatType, DoubleType, TimestampType)).collect(Collectors.toList()); - - - static Boolean isSupportedType(String valueType, Boolean forV2) { - if (forV2) { - return OpDataTypes.supportedTypesV2.contains(valueType); - } else { - return OpDataTypes.supportedTypes.contains(valueType); - } - } -} - -interface BaseOp { - void validate(); - - Object eval(EvalContext ctx); - - default Boolean evalExpectBoolean(EvalContext ctx) { - return (Boolean) eval(ctx); - } - - List getAllChildren(); - - default Boolean treeDepthExceeds(Integer depth) { - if (depth <= 0) { - return true; - } else { - return getAllChildren().stream().anyMatch(c -> c.treeDepthExceeds(depth - 1)); - } - } -} - - -// Represents a leaf operation. -abstract class LeafOp implements BaseOp { - - abstract Boolean isNull(EvalContext ctx); - - Pair evalExpectValueAndType(EvalContext ctx) { - return (Pair) eval(ctx); - } - - @Override - public List getAllChildren() { - return List.of(); - } - - abstract String getOpValueType(); -} - -// Represents a non-leaf operation. -abstract class NonLeafOp implements BaseOp { - List children; - - - public List getAllChildren() { - return (List) List.copyOf(children).stream().map(c -> (BaseOp) c); - } -} - -// Represents a unary operation. -interface UnaryOp { - // Validates number of children to be 1. - default void validateChildren(List children) { - if (children.size() != 1) - throw new IllegalArgumentException( - this + " : expected 1 but found " + children.size() + " children" - ); - children.get(0).validate(); - } -} - -interface BinaryOp { - // Validates number of children to be 2. - default void validateChildren(List children) { - if (children.size() != 2) - throw new IllegalArgumentException( - this + " : expected 2 but found " + children.size() + " children" - ); - children.forEach(BaseOp::validate); - var child1 = children.get(0); - var child2 = children.get(1); - if (child1 instanceof LeafOp && child2 instanceof LeafOp) { - var leftType = ((LeafOp) child1).getOpValueType(); - var rightType = ((LeafOp) child2).getOpValueType(); - if (!Objects.equals(leftType, rightType)) { - throw new IllegalArgumentException( - "Type mismatch: " + leftType + " vs " + rightType + " for " + - child1 + " and " + child2 - ); - } - } - } -} - -interface NaryOp { - // Validates number of children to be at least 2. - default void validateChildren(List children) { - if (children.size() < 2) { - throw new IllegalArgumentException( - this + " : expected at least 2 but found " + children.size() + " children" - ); - } - children.forEach(BaseOp::validate); - } -} - -class ColumnOp extends LeafOp { - - String name; - String valueType; - - public ColumnOp(String name, String valueType) { - this.name = name; - this.valueType = valueType; - } - - // Determine if the column value is null. - @Override - public Boolean isNull(EvalContext ctx) { - return resolve(ctx) == null; - } - - @Override - public Boolean evalExpectBoolean(EvalContext ctx) { - if (!Objects.equals(valueType, OpDataTypes.BoolType)) { - throw new IllegalArgumentException( - "Unsupported type for boolean evaluation: " + valueType - ); - } - return Boolean.valueOf(resolve(ctx)); - } - - @Override - public String getOpValueType() { - return null; - } - - @Override - public Object eval(EvalContext ctx) { - return Pair.of(resolve(ctx), valueType); - } - - public void validate() { - if (name == null) { - throw new IllegalArgumentException("Name must be specified: " + this); - } - if (!OpDataTypes.isSupportedType(valueType, false)) { - throw new IllegalArgumentException("Unsupported type: " + valueType); - } - } - - private String resolve(EvalContext ctx) { - return ctx.partitionValues.getOrDefault(name, null); - } -} - -class LiteralOp extends LeafOp { - - String value; - String valueType; - - @Override - public void validate() { - if (value == null) { - throw new IllegalArgumentException("Value must be specified: " + this); - } - if (!OpDataTypes.isSupportedType(valueType, false)) { - throw new IllegalArgumentException("Unsupported type: " + valueType); - } - EvalHelper.validateValue(value, valueType); - } - - @Override - public Object eval(EvalContext ctx) { - return Pair.of(value, valueType); - } - - public LiteralOp(String value, String valueType) { - this.value = value; - this.valueType = valueType; - } - - @Override - public Boolean isNull(EvalContext ctx) { - return false; - } - - - @Override - public String getOpValueType() { - return valueType; - } -} - -class IsNullOp extends NonLeafOp implements UnaryOp { - - public IsNullOp(List children) { - this.children = children; - } - - @Override - public void validate() { - - } - - @Override - public Object eval(EvalContext ctx) { - return children.get(0).isNull(ctx); - } - - -} - -class EqualOp extends NonLeafOp implements BinaryOp { - List children; - - public EqualOp(List children) { - this.children = children; - } - - @Override - public void validate() { - validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); - } - - @Override - public Object eval(EvalContext ctx) { - return EvalHelper.equal(children, ctx); - } -} - -class LessThanOp extends NonLeafOp implements BinaryOp { - List children; - - public LessThanOp(List children) { - this.children = children; - } - - @Override - public void validate() { - validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); - } - - @Override - public Object eval(EvalContext ctx) { - return EvalHelper.lessThan(children, ctx); - } -} - -class LessThanOrEqualOp extends NonLeafOp implements BinaryOp { - - public LessThanOrEqualOp(List children) { - this.children = children; - } - - @Override - public void validate() { - validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); - } - - @Override - public Object eval(EvalContext ctx) { - return EvalHelper.lessThan(children, ctx) || EvalHelper.equal(children, ctx); - } -} - - -class GreaterThanOp extends NonLeafOp implements BinaryOp { - - public GreaterThanOp(List children) { - this.children = children; - } - - @Override - public void validate() { - validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); - } - - @Override - public Object eval(EvalContext ctx) { - return !EvalHelper.lessThan(children, ctx) && !EvalHelper.equal(children, ctx); - } -} - -class GreaterThanOrEqualOp extends NonLeafOp implements BinaryOp { - - public GreaterThanOrEqualOp(List children) { - this.children = children; - } - - @Override - public void validate() { - validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); - } - - @Override - public Object eval(EvalContext ctx) { - return !EvalHelper.lessThan(children, ctx); - } -} - -class AndOp extends NonLeafOp implements BinaryOp { - - public AndOp(List children) { - this.children = children; - } - - @Override - public void validate() { - validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); - } - - @Override - public Object eval(EvalContext ctx) { - return children.stream().allMatch(c -> c.evalExpectBoolean(ctx)); - } -} - -class OrOp extends NonLeafOp implements BinaryOp { - - public OrOp(List children) { - this.children = children; - } - - @Override - public void validate() { - validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); - } - - @Override - public Object eval(EvalContext ctx) { - return children.stream().anyMatch(c -> c.evalExpectBoolean(ctx)); - } -} - -class NotOp extends NonLeafOp implements UnaryOp { - - public NotOp(List children) { - this.children = children; - } - - @Override - public void validate() { - validateChildren((List) List.copyOf(children).stream().map(c -> (BaseOp) c)); - } - - @Override - public Object eval(EvalContext ctx) { - return !children.get(0).evalExpectBoolean(ctx); - } -} \ No newline at end of file diff --git a/server/src/main/java/io/whitefox/core/PartitionFilterUtils.java b/server/src/main/java/io/whitefox/core/PartitionFilterUtils.java deleted file mode 100644 index 8f4cb3f0d..000000000 --- a/server/src/main/java/io/whitefox/core/PartitionFilterUtils.java +++ /dev/null @@ -1,5 +0,0 @@ -package io.whitefox.core; - -public class PartitionFilterUtils { - -} diff --git a/server/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/src/main/java/io/whitefox/core/types/predicates/BaseOp.java new file mode 100644 index 000000000..3ccdcfab1 --- /dev/null +++ b/server/src/main/java/io/whitefox/core/types/predicates/BaseOp.java @@ -0,0 +1,109 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.whitefox.core.types.*; + +import java.util.List; +import java.util.Objects; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + include = JsonTypeInfo.As.PROPERTY, + property = "op") +@JsonSubTypes({ + @JsonSubTypes.Type(value = LeafOp.class, names = {"column", "literal"}), +// @JsonSubTypes.Type(value = EqualOp.class, names = {"equal"/*, "not", "or", "and", "lessThan", "lessThanOrEqual", "greaterThan", "greaterThanOrEqualOp" */}), +// @JsonSubTypes.Type(value = ColumnOp.class, name = "column"), +// @JsonSubTypes.Type(value = LiteralOp.class, name = "literal"), + @JsonSubTypes.Type(value = EqualOp.class, name = "equal"), + @JsonSubTypes.Type(value = NotOp.class, name = "not"), + @JsonSubTypes.Type(value = OrOp.class, name = "or"), + @JsonSubTypes.Type(value = AndOp.class, name = "and"), + @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), + @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), + @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), + @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqualOp") +}) +public interface BaseOp { + + void validate(); + + default Boolean isSupportedType(DataType valueType, Boolean forV2) { + if (forV2) { + return (valueType instanceof BooleanType + || valueType instanceof IntegerType + || valueType instanceof StringType + || valueType instanceof DateType + || valueType instanceof LongType + || valueType instanceof TimestampType + || valueType instanceof FloatType + || valueType instanceof DoubleType); + } else { + return (valueType instanceof BooleanType + || valueType instanceof IntegerType + || valueType instanceof StringType + || valueType instanceof DateType + || valueType instanceof LongType); + } + } + + Object eval(EvalContext ctx); + + default Boolean evalExpectBoolean(EvalContext ctx) { + return (Boolean) eval(ctx); + } + + List getAllChildren(); + + default Boolean treeDepthExceeds(Integer depth) { + if (depth <= 0) { + return true; + } else { + return getAllChildren().stream().anyMatch(c -> c.treeDepthExceeds(depth - 1)); + } + } +} + + +// Represents a unary operation. +interface UnaryOp { + // Validates number of children to be 1. + default void validateChildren(List children) { + if (children.size() != 1) + throw new IllegalArgumentException( + this + " : expected 1 but found " + children.size() + " children"); + children.get(0).validate(); + } +} + +interface BinaryOp { + // Validates number of children to be 2. + default void validateChildren(List children) { + if (children.size() != 2) + throw new IllegalArgumentException( + this + " : expected 2 but found " + children.size() + " children"); + children.forEach(BaseOp::validate); + var child1 = children.get(0); + var child2 = children.get(1); + if (child1 instanceof LeafOp && child2 instanceof LeafOp) { + var leftType = ((LeafOp) child1).getOpValueType(); + var rightType = ((LeafOp) child2).getOpValueType(); + if (!Objects.equals(leftType, rightType)) { + throw new IllegalArgumentException("Type mismatch: " + leftType + " vs " + rightType + + " for " + child1 + " and " + child2); + } + } + } +} + +interface NaryOp { + // Validates number of children to be at least 2. + default void validateChildren(List children) { + if (children.size() < 2) { + throw new IllegalArgumentException( + this + " : expected at least 2 but found " + children.size() + " children"); + } + children.forEach(BaseOp::validate); + } +} diff --git a/server/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java b/server/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java new file mode 100644 index 000000000..7f560908c --- /dev/null +++ b/server/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java @@ -0,0 +1,46 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import io.whitefox.core.types.*; + +import java.io.IOException; + +public class DataTypeDeserializer extends StdDeserializer { + + // needed for jackson + public DataTypeDeserializer() { + this(null); + } + + public DataTypeDeserializer(Class vc) { + super(vc); + } + + @Override + public DataType deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + String valueType = node.asText(); + + switch (valueType) { + case "date": + return DateType.DATE; + case "int": + return IntegerType.INTEGER; + case "double": + return DoubleType.DOUBLE; + case "float": + return FloatType.FLOAT; + case "string": + return StringType.STRING; + case "timestamp": + return TimestampType.TIMESTAMP; + case "long": + return LongType.LONG; + default: + throw new IOException("Unknown type passed inside a json predicate: " + valueType); + } + } +} \ No newline at end of file diff --git a/server/src/main/java/io/whitefox/core/types/predicates/EvalContext.java b/server/src/main/java/io/whitefox/core/types/predicates/EvalContext.java new file mode 100644 index 000000000..3320d4eee --- /dev/null +++ b/server/src/main/java/io/whitefox/core/types/predicates/EvalContext.java @@ -0,0 +1,17 @@ +package io.whitefox.core.types.predicates; + +import org.apache.commons.lang3.tuple.Pair; + +import java.util.Map; + +class EvalContext { + + public EvalContext( + Map partitionValues, Map> statsValues) { + this.partitionValues = partitionValues; + this.statsValues = statsValues; + } + + Map partitionValues; + Map> statsValues; +} \ No newline at end of file diff --git a/server/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java new file mode 100644 index 000000000..0da4645af --- /dev/null +++ b/server/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -0,0 +1,105 @@ +package io.whitefox.core.types.predicates; + +import io.whitefox.core.types.*; +import java.sql.Date; +import java.sql.Timestamp; +import java.util.List; +import java.util.Objects; + +import io.whitefox.core.types.predicates.LeafOp; +import org.apache.commons.lang3.tuple.Pair; + +public class EvalHelper { + + static Pair, Pair> validateAndGetTypeAndValue( + List children, EvalContext ctx) { + var leftChild = children.get(0); + var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); + var leftVal = leftChild.evalExpectValueAndType(ctx).getLeft(); + + var rightChild = children.get(1); + var rightType = rightChild.evalExpectValueAndType(ctx).getRight(); + var rightVal = rightChild.evalExpectValueAndType(ctx).getLeft(); + // If the types don't match, it implies a malformed predicate tree. + // We simply throw an exception, which will cause filtering to be skipped. + if (!Objects.equals(leftType, rightType)) { + throw new IllegalArgumentException("Type mismatch: " + leftType + " vs " + rightType + " for " + + leftChild + " and " + rightChild); + } + + // We throw an exception for nulls, which will skip filtering. + if (leftVal == null || rightVal == null) { + throw new IllegalArgumentException( + "Comparison with null is not supported: " + leftChild + " and " + rightChild); + } + return Pair.of(Pair.of(leftType, leftVal), Pair.of(rightType, rightVal)); + } + + // Implements "equal" between two leaf operations. + static Boolean equal(List children, EvalContext ctx) { + var typesAndValues = validateAndGetTypeAndValue(children, ctx); + var leftType = typesAndValues.getLeft().getLeft(); + var leftVal = typesAndValues.getLeft().getRight(); + var rightVal = typesAndValues.getRight().getRight(); + + if (BooleanType.BOOLEAN.equals(leftType)) { + return Boolean.valueOf(leftVal) == Boolean.valueOf(rightVal); + } else if (IntegerType.INTEGER.equals(leftType)) { + return Integer.parseInt(leftVal) == Integer.parseInt(rightVal); + } else if (LongType.LONG.equals(leftType)) { + return Long.parseLong(leftVal) == Long.parseLong(rightVal); + } else if (StringType.STRING.equals(leftType)) { + return leftVal.equals(rightVal); + } else if (DateType.DATE.equals(leftType)) { + return Date.valueOf(leftVal).equals(Date.valueOf(rightVal)); + } + throw new IllegalArgumentException("Unsupported type: " + leftType); + } + + // TODO: supported expressions; ie. check if column + constant + // TODO: handle column comparisons with literals + static Boolean lessThan(List children, EvalContext ctx) { + var typesAndValues = validateAndGetTypeAndValue(children, ctx); + var leftType = typesAndValues.getLeft().getLeft(); + var leftVal = typesAndValues.getLeft().getRight(); + var rightVal = typesAndValues.getRight().getRight(); + + if (IntegerType.INTEGER.equals(leftType)) { + return Integer.parseInt(leftVal) < Integer.parseInt(rightVal); + } else if (LongType.LONG.equals(leftType)) { + return Long.parseLong(leftVal) < Long.parseLong(rightVal); + } else if (StringType.STRING.equals(leftType)) { + return leftVal.compareTo(rightVal) < 0; + } else if (DateType.DATE.equals(leftType)) { + return Date.valueOf(leftVal).before(Date.valueOf(rightVal)); + } + throw new IllegalArgumentException("Unsupported type: " + leftType); + } + // Validates that the specified value is in the correct format. + // Throws an exception otherwise. + public static void validateValue(String value, DataType valueType) { + try { + if (BooleanType.BOOLEAN.equals(valueType)) { + Boolean.parseBoolean(value); + } else if (IntegerType.INTEGER.equals(valueType)) { + Integer.parseInt(value); + } else if (LongType.LONG.equals(valueType)) { + Long.parseLong(value); + } else if (DateType.DATE.equals(valueType)) { + Date.valueOf(value); + } else if (FloatType.FLOAT.equals(valueType)) { + Float.parseFloat(value); + } else if (DoubleType.DOUBLE.equals(valueType)) { + Double.parseDouble(value); + // TODO check for non deprecated + } else if (TimestampType.TIMESTAMP.equals(valueType)) { + Timestamp.valueOf(value); + } else { + throw new IllegalArgumentException("Unsupported type: " + valueType); + } + } catch (Exception e) { + throw new IllegalArgumentException( + "Error validating " + value + " for type " + valueType + ": " + e); + } + } +} diff --git a/server/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/src/main/java/io/whitefox/core/types/predicates/LeafOp.java new file mode 100644 index 000000000..64a4c8cd3 --- /dev/null +++ b/server/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -0,0 +1,142 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import io.whitefox.core.types.BooleanType; +import io.whitefox.core.types.DataType; +import org.apache.commons.lang3.tuple.Pair; + +//import java.beans.ConstructorProperties; +import java.util.List; +import java.util.Objects; + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "op" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = ColumnOp.class, name = "column"), + @JsonSubTypes.Type(value = LiteralOp.class, name = "literal")} +) +//@JsonDeserialize(using = LeafOpDeserializer.class) +public abstract class LeafOp implements BaseOp { + + abstract Boolean isNull(EvalContext ctx); + + @JsonDeserialize(using = DataTypeDeserializer.class) + @JsonProperty("valueType") + DataType valueType; + + Pair evalExpectValueAndType(EvalContext ctx) { + return (Pair) eval(ctx); + } + + @Override + public List getAllChildren() { + return List.of(); + } + + abstract DataType getOpValueType(); +} + + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "column") +class ColumnOp extends LeafOp { + + @JsonProperty("name") + String name; + + public ColumnOp(){ + super(); + } + + public ColumnOp(String name, DataType valueType) { + this.name = name; + this.valueType = valueType; + } + + // Determine if the column value is null. + @Override + public Boolean isNull(EvalContext ctx) { + return resolve(ctx) == null; + } + + @Override + public Boolean evalExpectBoolean(EvalContext ctx) { + if (!Objects.equals(valueType, BooleanType.BOOLEAN)) { + throw new IllegalArgumentException("Unsupported type for boolean evaluation: " + valueType); + } + return Boolean.valueOf(resolve(ctx)); + } + + @Override + public DataType getOpValueType() { + return valueType; + } + + @Override + public Object eval(EvalContext ctx) { + return Pair.of(resolve(ctx), valueType); + } + + public void validate() { + if (name == null) { + throw new IllegalArgumentException("Name must be specified: " + this); + } + if (!this.isSupportedType(valueType, false)) { + throw new IllegalArgumentException("Unsupported type: " + valueType); + } + } + + private String resolve(EvalContext ctx) { + return ctx.partitionValues.getOrDefault(name, null); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "literal") +class LiteralOp extends LeafOp { + @JsonProperty("value") + String value; + + @Override + public void validate() { + if (value == null) { + throw new IllegalArgumentException("Value must be specified: " + this); + } + if (!isSupportedType(valueType, false)) { + throw new IllegalArgumentException("Unsupported type: " + valueType); + } + EvalHelper.validateValue(value, valueType); + } + + @Override + public Object eval(EvalContext ctx) { + return Pair.of(value, valueType); + } + + public LiteralOp() { + super(); + } + + public LiteralOp(String value, DataType valueType) { + this.value = value; + this.valueType = valueType; + } + + @Override + public Boolean isNull(EvalContext ctx) { + return false; + } + + @Override + public DataType getOpValueType() { + return valueType; + } +} \ No newline at end of file diff --git a/server/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java b/server/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java new file mode 100644 index 000000000..4e4824e00 --- /dev/null +++ b/server/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java @@ -0,0 +1,39 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import io.whitefox.core.types.DateType; + +import java.io.IOException; + +public class LeafOpDeserializer extends StdDeserializer { + public LeafOpDeserializer() { + this(null); + } + + public LeafOpDeserializer(Class vc) { + super(vc); + } + + @Override + public LeafOp deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + String valueType = node.get("valueType").asText(); + String op = node.get("op").asText(); + + // For example: + switch (op) { + case "column": + String name = node.get("name").asText(); +// switch (valueType) + return new ColumnOp(name, DateType.DATE); + case "literal": + String value = node.get("value").asText(); + return new LiteralOp(value, DateType.DATE); + default: + throw new IOException("type not known"); + } + } +} \ No newline at end of file diff --git a/server/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java new file mode 100644 index 000000000..ac47252d3 --- /dev/null +++ b/server/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java @@ -0,0 +1,259 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import java.util.List; +import java.util.stream.Collectors; + +// Represents a non-leaf operation. +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "op" +) +@JsonSubTypes({ + @JsonSubTypes.Type(value = EqualOp.class, name = "equal"), + @JsonSubTypes.Type(value = NotOp.class, name = "not"), + @JsonSubTypes.Type(value = OrOp.class, name = "or"), + @JsonSubTypes.Type(value = IsNullOp.class, name = "null"), + @JsonSubTypes.Type(value = AndOp.class, name = "and"), + @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), + @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), + @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), + @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqualOp") +}) +public abstract class NonLeafOp implements BaseOp { + + @JsonProperty("children") + List children; + + public List getAllChildren() { + // TODO flat map every child + return List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList()); + } +} + +class IsNullOp extends NonLeafOp implements UnaryOp { + + @JsonProperty("children") + List children; + + public IsNullOp(List children) { + this.children = children; + } + + public IsNullOp() { + super(); + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return ((LeafOp) children.get(0)).isNull(ctx); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "equal") +class EqualOp extends NonLeafOp implements BinaryOp { + @JsonProperty("children") + List children; + + public EqualOp() { + super(); + } + + public EqualOp(List children) { + this.children = children; + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return EvalHelper.equal(children, ctx); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "lessThan") +class LessThanOp extends NonLeafOp implements BinaryOp { + @JsonProperty("children") + List children; + + public LessThanOp(List children) { + this.children = children; + } + + public LessThanOp() { + super(); + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return EvalHelper.lessThan(children, ctx); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "lessThanOrEqual") +class LessThanOrEqualOp extends NonLeafOp implements BinaryOp { + + @JsonProperty("children") + List children; + + public LessThanOrEqualOp(List children) { + this.children = children; + } + + public LessThanOrEqualOp() { + super(); + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return EvalHelper.lessThan(children, ctx) || EvalHelper.equal(children, ctx); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "greaterThan") +class GreaterThanOp extends NonLeafOp implements BinaryOp { + + @JsonProperty("children") + List children; + + public GreaterThanOp() { + super(); + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return !EvalHelper.lessThan(children, ctx) && !EvalHelper.equal(children, ctx); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "greaterThanOrEqual") +class GreaterThanOrEqualOp extends NonLeafOp implements BinaryOp { + + @JsonProperty("children") + List children; + + public GreaterThanOrEqualOp() { + super(); + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return !EvalHelper.lessThan(children, ctx); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "and") +class AndOp extends NonLeafOp implements BinaryOp { + + public AndOp(List children) { + this.children = children; + } + + public AndOp() { + super(); + } + + @Override + public void validate() { + validateChildren(children); + } + + @Override + public Object eval(EvalContext ctx) { + return children.stream().allMatch(c -> c.evalExpectBoolean(ctx)); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "or") +class OrOp extends NonLeafOp implements BinaryOp { + + public OrOp(List children) { + this.children = children; + } + + public OrOp() { + super(); + } + + @Override + public void validate() { + validateChildren(children); + } + + @Override + public Object eval(EvalContext ctx) { + return children.stream().anyMatch(c -> c.evalExpectBoolean(ctx)); + } +} + +@JsonTypeInfo( + use = JsonTypeInfo.Id.NAME, + property = "not") +class NotOp extends NonLeafOp implements UnaryOp { + + @JsonProperty("children") + List children; + + public NotOp(List children) { + this.children = children; + } + + public NotOp() { + super(); + } + + @Override + public void validate() { + validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) { + return !children.get(0).evalExpectBoolean(ctx); + } +} \ No newline at end of file From b33ac8b2b418167e45d4f60524eb23a444c8d40a Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 24 Nov 2023 13:46:51 +0100 Subject: [PATCH 03/30] wip - partition columns filterable --- .../api/core/JsonPredicatesUtilsTest.java | 30 ++ .../java/io/whitefox/core/ColumnRange.java | 34 +++ .../main/java/io/whitefox/core/FileStats.java | 47 ++++ .../io/whitefox/core/JsonPredicatesUtils.java | 41 +++ .../core/services/DeltaSharedTable.java | 18 +- .../core/types/predicates/BaseOp.java | 1 - .../core/types/predicates/EvalContext.java | 14 +- .../core/types/predicates/EvalHelper.java | 2 +- .../core/types/predicates/LeafOp.java | 1 - .../core/types/predicates/NonLeafOp.java | 1 + .../core/services/DeltaSharedTableTest.java | 2 +- .../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 44 bytes .../_delta_log/.00000000000000000001.json.crc | Bin 0 -> 36 bytes .../_delta_log/00000000000000000000.json | 12 + .../_delta_log/00000000000000000001.json | 12 + ...-86d8-01231ba570e7.c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...-8843-1f6b1e3bc4f6.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-a091-4896b9f69ec8.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-b491-1a44b562fa3c.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-8466-2ce9f04cb3ef.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-bc6c-937559d2c39a.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...4f20-86d8-01231ba570e7.c000.snappy.parquet | Bin 0 -> 1044 bytes ...4ad0-8843-1f6b1e3bc4f6.c000.snappy.parquet | Bin 0 -> 937 bytes ...4751-a091-4896b9f69ec8.c000.snappy.parquet | Bin 0 -> 937 bytes ...453c-b491-1a44b562fa3c.c000.snappy.parquet | Bin 0 -> 937 bytes ...435c-8466-2ce9f04cb3ef.c000.snappy.parquet | Bin 0 -> 937 bytes ...45f8-bc6c-937559d2c39a.c000.snappy.parquet | Bin 0 -> 937 bytes ...-a7a7-3115e28814b7.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-bf4b-cfbd789ff11c.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-bded-b62c066e32c9.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-8e25-44a270e58690.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...-984b-365fde6e183a.c000.snappy.parquet.crc | Bin 0 -> 16 bytes ...41c3-a7a7-3115e28814b7.c000.snappy.parquet | Bin 0 -> 1003 bytes ...4e37-bf4b-cfbd789ff11c.c000.snappy.parquet | Bin 0 -> 937 bytes ...4380-bded-b62c066e32c9.c000.snappy.parquet | Bin 0 -> 936 bytes ...4bf5-8e25-44a270e58690.c000.snappy.parquet | Bin 0 -> 937 bytes ...4d3b-984b-365fde6e183a.c000.snappy.parquet | Bin 0 -> 937 bytes .../core/types/predicates/BaseOp.java | 1 - .../core/types/predicates/EvalContext.java | 17 -- .../core/types/predicates/EvalHelper.java | 105 ------- .../core/types/predicates/LeafOp.java | 1 - .../core/types/predicates/NonLeafOp.java | 259 ------------------ 42 files changed, 202 insertions(+), 396 deletions(-) create mode 100644 server/app/src/test/java/io/whitefox/api/core/JsonPredicatesUtilsTest.java create mode 100644 server/core/src/main/java/io/whitefox/core/ColumnRange.java create mode 100644 server/core/src/main/java/io/whitefox/core/FileStats.java create mode 100644 server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/.00000000000000000000.json.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/.00000000000000000001.json.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/00000000000000000000.json create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/00000000000000000001.json create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/.part-00000-1dcbbcb3-4245-4f20-86d8-01231ba570e7.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/.part-00001-6e82ece6-3597-4ad0-8843-1f6b1e3bc4f6.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/.part-00003-86eb45e1-a91f-4751-a091-4896b9f69ec8.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/.part-00004-80d754aa-1567-453c-b491-1a44b562fa3c.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/.part-00006-f290581b-367f-435c-8466-2ce9f04cb3ef.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/.part-00007-1714c8b5-a279-45f8-bc6c-937559d2c39a.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00000-1dcbbcb3-4245-4f20-86d8-01231ba570e7.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00001-6e82ece6-3597-4ad0-8843-1f6b1e3bc4f6.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00003-86eb45e1-a91f-4751-a091-4896b9f69ec8.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00004-80d754aa-1567-453c-b491-1a44b562fa3c.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00006-f290581b-367f-435c-8466-2ce9f04cb3ef.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00007-1714c8b5-a279-45f8-bc6c-937559d2c39a.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00001-f38b954d-41f3-41c3-a7a7-3115e28814b7.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00009-dc49acd6-1a95-4e37-bf4b-cfbd789ff11c.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00011-f2482f68-663e-4380-bded-b62c066e32c9.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00013-fd83bc7c-cb1e-4bf5-8e25-44a270e58690.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00015-78628903-d770-4d3b-984b-365fde6e183a.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00001-f38b954d-41f3-41c3-a7a7-3115e28814b7.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00009-dc49acd6-1a95-4e37-bf4b-cfbd789ff11c.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00011-f2482f68-663e-4380-bded-b62c066e32c9.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00013-fd83bc7c-cb1e-4bf5-8e25-44a270e58690.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00015-78628903-d770-4d3b-984b-365fde6e183a.c000.snappy.parquet delete mode 100644 server/src/main/java/io/whitefox/core/types/predicates/EvalContext.java delete mode 100644 server/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java delete mode 100644 server/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java diff --git a/server/app/src/test/java/io/whitefox/api/core/JsonPredicatesUtilsTest.java b/server/app/src/test/java/io/whitefox/api/core/JsonPredicatesUtilsTest.java new file mode 100644 index 000000000..e5a9cdb5a --- /dev/null +++ b/server/app/src/test/java/io/whitefox/api/core/JsonPredicatesUtilsTest.java @@ -0,0 +1,30 @@ +package io.whitefox.core; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.delta.standalone.DeltaLog; +import io.whitefox.core.services.DeltaSharedTable; +import io.whitefox.core.types.predicates.BaseOp; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Test; + +import java.util.stream.Collectors; + +import static io.whitefox.api.server.DeltaTestUtils.deltaTable; +import static io.whitefox.api.server.DeltaTestUtils.deltaTableUri; + +public class JsonPredicatesUtilsTest { + + @Test + void testCreateEvalContext() { + var PTable = new SharedTable( + "partitioned-delta-table-with-multiple-columns", "default", "share1", deltaTable("partitioned-delta-table-with-multiple-columns")); + + var log = DeltaLog.forTable(new Configuration(), deltaTableUri("partitioned-delta-table-with-multiple-columns")); + var contexts = log.snapshot().getAllFiles().stream().map(JsonPredicatesUtils::createEvalContext).collect(Collectors.toList()); + assert(contexts.size()==2); + var c1 = contexts.get(0); + assert(c1.getPartitionValues().get("date").equals("2021-08-09")); + + } +} diff --git a/server/core/src/main/java/io/whitefox/core/ColumnRange.java b/server/core/src/main/java/io/whitefox/core/ColumnRange.java new file mode 100644 index 000000000..9487307b0 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/ColumnRange.java @@ -0,0 +1,34 @@ +package io.whitefox.core; + +import java.util.Comparator; + +public class ColumnRange{ + + T minVal; + T maxVal; + private final Comparator ord; + + public ColumnRange(T minVal, T maxVal, Comparator ord) { + this.minVal = minVal; + this.maxVal = maxVal; + this.ord = ord; + } + + public ColumnRange(T onlyVal, Comparator ord) { + this.minVal = onlyVal; + this.maxVal = onlyVal; + this.ord = ord; + } + + public Boolean contains(T point) { + var c1 = ord.compare(minVal, point); + var c2 = ord.compare(maxVal, point); + return (c1 <= 0 && c2 >= 0); + } + + public static ColumnRange toLong(String minVal, String maxVal) { + return new ColumnRange<>(Long.getLong(minVal), Long.getLong(maxVal), Comparator.naturalOrder()); + } + + +} diff --git a/server/core/src/main/java/io/whitefox/core/FileStats.java b/server/core/src/main/java/io/whitefox/core/FileStats.java new file mode 100644 index 000000000..dab50fa7d --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/FileStats.java @@ -0,0 +1,47 @@ +package io.whitefox.core; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.Map; + +public class FileStats { + //{"numRecords":1,"minValues":{"id":0},"maxValues":{"id":0},"nullCount":{"id":0}} + @JsonProperty("numRecords") + String numRecords; + + @JsonProperty("minValues") + Map minValues; + + @JsonProperty("maxValues") + Map maxValues; + + @JsonProperty("nullCount") + Map nullCount; + + public FileStats() { + super(); + } + + public String getNumRecords() { + return numRecords; + } + + public Map getMinValues() { + return minValues; + } + + public Map getMaxValues() { + return maxValues; + } + + public Map getNullCount() { + return nullCount; + } + + public FileStats(String numRecords, Map minValues, Map maxValues, Map nullCount) { + this.numRecords = numRecords; + this.minValues = minValues; + this.maxValues = maxValues; + this.nullCount = nullCount; + } +} diff --git a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java new file mode 100644 index 000000000..bfc60ee02 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java @@ -0,0 +1,41 @@ +package io.whitefox.core; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.delta.standalone.actions.AddFile; +import io.whitefox.core.types.predicates.BaseOp; +import io.whitefox.core.types.predicates.EvalContext; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.Comparator; +import java.util.Map; + +public class JsonPredicatesUtils { + + public static BaseOp parsePredicate(String predicate) throws JsonProcessingException { + var mapper = new ObjectMapper(); + return mapper.readValue(predicate, BaseOp.class); + } + + public static EvalContext createEvalContext(AddFile file){ + var statsString = file.getStats(); + var partitionValues = file.getPartitionValues(); + + var mapper = new ObjectMapper(); + try { + var fileStats = mapper.readValue(statsString, FileStats.class); + var maxValues = fileStats.maxValues; + var mappedMinMaxPairs = new java.util.HashMap>(); + fileStats.getMinValues().forEach((minK,minV) -> { + String maxV = maxValues.get(minK); + Pair minMaxPair = Pair.of(minV, maxV); + mappedMinMaxPairs.put(minK, minMaxPair); + }); + return new EvalContext(partitionValues, mappedMinMaxPairs); + } + catch (JsonProcessingException e) { + var message = e.getMessage(); + return new EvalContext(partitionValues, Map.of()); + } + } +} diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index f8ec7726a..2db756d66 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -94,11 +94,17 @@ public static BaseOp parsePredicate(String predicate) throws JsonProcessingExcep } } - public boolean filterFilesBasedOnPredicates(List predicates, AddFile f) { - var partitionValues = f.getPartitionValues(); - predicates.forEach(p -> {}); - - return true; + public boolean filterFileBasedOnPredicates(List predicates, AddFile f) { + var ctx = JsonPredicatesUtils.createEvalContext(f); + return predicates.stream().allMatch(p -> { + try { + var parsedPredicate = JsonPredicatesUtils.parsePredicate(p); + return parsedPredicate.evalExpectBoolean(ctx); + } catch (JsonProcessingException e) { + System.out.println("Unable to parse predicate: " + p +" due to: " + e); + return false; + } + }); } public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { @@ -121,7 +127,7 @@ public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { return new ReadTableResultToBeSigned( new Protocol(Optional.of(1)), metadataFromSnapshot(snapshot), - snapshot.getAllFiles().stream() + snapshot.getAllFiles().stream().filter(f -> filterFileBasedOnPredicates(predicates, f)) .map(f -> new TableFileToBeSigned( location() + "/" + f.getPath(), f.getSize(), diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java index 3ccdcfab1..74a928bc1 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.whitefox.core.types.*; import java.util.List; import java.util.Objects; diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalContext.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalContext.java index 3320d4eee..a454190ca 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalContext.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalContext.java @@ -4,7 +4,7 @@ import java.util.Map; -class EvalContext { +public class EvalContext { public EvalContext( Map partitionValues, Map> statsValues) { @@ -12,6 +12,14 @@ public EvalContext( this.statsValues = statsValues; } - Map partitionValues; - Map> statsValues; + final Map partitionValues; + final Map> statsValues; + + public Map getPartitionValues() { + return partitionValues; + } + + public Map> getStatsValues() { + return statsValues; + } } \ No newline at end of file diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index 0da4645af..0f431d17a 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -6,9 +6,9 @@ import java.util.List; import java.util.Objects; -import io.whitefox.core.types.predicates.LeafOp; import org.apache.commons.lang3.tuple.Pair; +// Only for partition values public class EvalHelper { static Pair, Pair> validateAndGetTypeAndValue( diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java index 64a4c8cd3..26de51e29 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -1,6 +1,5 @@ package io.whitefox.core.types.predicates; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java index ac47252d3..478e49dee 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java @@ -203,6 +203,7 @@ public void validate() { @Override public Object eval(EvalContext ctx) { + // short-circuits, so not all exceptions will be thrown return children.stream().allMatch(c -> c.evalExpectBoolean(ctx)); } } diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index f6a519e30..f961cdb3c 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -77,7 +77,7 @@ void getTableVersionWithMalformedTimestamp() throws ExecutionException, Interrup } @Test - void queryTable() throws ExecutionException, InterruptedException { + void queryTable() { var PTable = new SharedTable( "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/.00000000000000000000.json.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/.00000000000000000000.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..f996377c00ddb35e9dd94f31915f2ba0b238036a GIT binary patch literal 44 zcmV+{0Mq|ta$^7h00ICmPgH%pzr~7GmYKdsnTz7UakL9k~Q; literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/.part-00006-f290581b-367f-435c-8466-2ce9f04cb3ef.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/.part-00006-f290581b-367f-435c-8466-2ce9f04cb3ef.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..067a6a90323262f43cb2d77b66de1f4cdfa7948b GIT binary patch literal 16 XcmYc;N@ieSU}A9JK3UXlY2+IKA|(Y8 literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/.part-00007-1714c8b5-a279-45f8-bc6c-937559d2c39a.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/.part-00007-1714c8b5-a279-45f8-bc6c-937559d2c39a.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..4e609f8a44ff85b04f9bc12de78c949d69af6527 GIT binary patch literal 16 XcmYc;N@ieSU}A`Rs?or%Ri^_0Ah84e literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00000-1dcbbcb3-4245-4f20-86d8-01231ba570e7.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00000-1dcbbcb3-4245-4f20-86d8-01231ba570e7.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ecf13bf370ce7867e5b2a2c9f01280f961d2baf9 GIT binary patch literal 1044 zcma)6&1(}u6rateC4}@KbcS6B7?w1okXd$iX1_`hycO|Kyorj;?#$L;KH7X#q_k)8 z;Ki#4{{q3Ah<|{mS_*>ygy_+mzHMSt@L=Y!`{uph`})nB*}Hv@VT2#zUHs$w$1iJj zT!rmAt{~)EC4`W96KxpC*r*u9FuEpaidCNf5MbY!_^pY5z5o5ScFp?8IQ1fr`J5Yj zKPOS{c?oBda^GWAq&%fk#vxT{C`6hiUXaKlte#m zd0gO>3Yf@)faXb{0mm3k6nvsct4#Tli72selxzz~nJ*Bp7%15Up-Xdfl|F5&xk_Dl z9=1*@_z2=7yG~B=g;|JT>h9Ji2Xg%M@g#eq2NIVNE^n2rQcodV1Hqj2)E2s12Q;bL zb-UB5)N$DkP-WdNP`E^@S8CTV&h*qBkDj>$J(WsMrS;l4cBan{w6o(( zrsG*QbvjPo(|t8@b{-r%Lpdl+Jq79otUns=!)`e1_hs4#E4b;7^EaZib9{2rx$*?Y zmadSY_kZU{C&;?9D3%V!jYW42<}%;)8|`$~>#GJ#FZZ$-ZVHb>odtfzvs{K57ipy7 ZZuyM+ewK)By90AljLLmD*gbd;{s6PW>r4Ou literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00001-6e82ece6-3597-4ad0-8843-1f6b1e3bc4f6.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00001-6e82ece6-3597-4ad0-8843-1f6b1e3bc4f6.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..8d9f1be5a7da1014fc6ec5e3d2a589b33533a9e1 GIT binary patch literal 937 zcma)5J&)5s5M6J=$wfdyx?S0l3(LB*?j&awdy_bcA|yI=LZ=f&8bGjFZw~96kFy;R zic-;0q@kuk)KpONA1El1Xzn-A!Z;T$ktpyM*6hriH#5((XZIiZ1QCna33*OdKYzW)J23Y78g>AnP9VCzFGbxCTfE-nfv9(6#OvL# zISQ5VgSIctyVx{<{`%~g1e+A0b@-qZkx>e^p|z+^PJthp79Eh4R+td#Kd5ChkzPE@ z4!O5F0p-5UGJtna9x~dT0TfV$(7`SGP#jY$1KmEKmefOi$>ktM|75x6(7WI8#A+hm?3 zVwgY_(v;`?WpsLr<t r>@>QZciKv}0?Fk_v?Xs2Lp9<&@Oco(Zu8jmVC9Of`v^;N29NX)^E1z- literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00003-86eb45e1-a91f-4751-a091-4896b9f69ec8.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00003-86eb45e1-a91f-4751-a091-4896b9f69ec8.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..e897b064436808bdc655347655e59cbed6ce271e GIT binary patch literal 937 zcma)5&1%~~5MHg_AgC!6vWo;d7?FqsunhO`WLnb5a06zcYMiM(Nw7{4T8H;45owiRC9OeKat!=PH|U65DuoH5;gd!-7wOwq zS@=dy1gRrZ-^Ut`BSact)E05HkDeNUrdT(0+fofu*FB`}=mm;kis=zOw$$NfsEtL@ z!QH4EFZ2LnLU|y)s*zoKqmW0)8X2TfG8bjn)bAB~saT_7wOOm7ps7mARb$&8=B7p{ z{fNnD1<{`h#g*0foA%EVlN%xtYVdneN@o=vrToN?Ah+TlNl>?~pMIA;r) zvvhXOCNdY1$c6SnS(^F9T$%$j%hN@eo3=TQWfEoP;Q7*=ib)ZQBXF0nWHvp8+jNm6 zB1j+#Y0@@t8O_0RwX*LrLH$iCREYnd{Bnh~qq5o7ByE<{H67--%xxY7i#UmzzAt$@ s?6v!x_qsCl+>nQ3(G7Vg@a33u*Ws=k_B)5V4J%h{-6vR*6ZlF007?wdRsaA1 literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00004-80d754aa-1567-453c-b491-1a44b562fa3c.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00004-80d754aa-1567-453c-b491-1a44b562fa3c.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c206488da29f2163a5c01ac2000d6234fb37e019 GIT binary patch literal 937 zcma)5&1%~~5MHgFpo1wCGK&N{7?GF+BA>YAYefiu0MT$zheQ# z>n`Yo+-p%nNJ-1T`9c3PcnQCMUH{a!VEoHp*gitqy=D-4p{RF5>DOCcC#d(^eOdS0 zBJjLcPc>xEyiZL-za`x=ftrkvRd~M;kx>Y?rnQ(3C&*7si;dt)%S{XoA63$sOkTf8 z{a0!%p-iBBKn)rN1S)8h=237!9$ScJ)U-^erdv=p+eF_ra}-cU*)cn*>BG%X8;g;{ zyHPh@>;c8Z@<{r%1bgg8g9oHGvY@hN&daW-FEw_h)kec&vlc^PQ-zc(#B64KaUO~SbgyCYbaIBb$vlpw z7h@FDlw;pA+JnV%>D*t^fYq`IE2+T=}_yVXN)9>qbm+f}0B s_Zod6dL8ArU0?X2?D(SPbyX;Y+Z3+r_ghD%gDaPB-A7!KQ~XK)05b2`PXGV_ literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00006-f290581b-367f-435c-8466-2ce9f04cb3ef.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00006-f290581b-367f-435c-8466-2ce9f04cb3ef.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..bfbdaa50349f8900a5e31ab58986039949f7d5b3 GIT binary patch literal 937 zcma)5zi-n}5Pp6Rm4^rksrO_{9wMuaTFH5e{gOC|qDpM2LRH0(1rR*{NU?B!)OLzc zl#Pj@OZf{BVq}7m|A7HE#{LUPjGRN-L}I{iui(rY%P z^u3zWA_qfN9Oqd);=lpV8^x;EGhwXqmE zxE*!l#U4;hERUq$RM=(L3LKI8(1hxmxhT7$eygxcr9K>#o3$JYn<}MTIkxR#u2o>{ zJ%vvSVc!)->l*v1?4stju20i5E@u*D<}1bdMU-aobfVu8T<|8*5C1`A=LyHhIiE*4 zPp7Z=ILc)xbE#h`3oCy;i>$tt<>@@gE!!H!Q4(fW|M9|_$Z-*hLv*iU$#imtx5+$7 zq@Q3E(<0ltVYK>-<*MD;hRSRwv@^3x^JkJrsM#%XOmUCrT+%iY?6KaZ2J=6R85 t2Hj>)h;Ane+HN3%k?aJb<$KXc2-gv=8}wR-hK(y%Y~2T3l2iOi{{W*M&vyU- literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00007-1714c8b5-a279-45f8-bc6c-937559d2c39a.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-09/part-00007-1714c8b5-a279-45f8-bc6c-937559d2c39a.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..dad5f92bc7360e0a8c80972081fa322445f31258 GIT binary patch literal 937 zcma)5&1%~~5MHg_po0k%GK&N{7?GF+0XIufI?W zC|vw$1z0H1u224HKxz2w8@AOA#5RU@KaS>2Qkt#I)D|F15nMQ2${qo67Xn z^DKC&MiQz7s(aL+aY&$sMr9U&V&J5_ei&hm5?v29aft2}ESZeY@iv|% zi5w;v#WdyEw~TggzF0VSnW+9c6)VL5Pky!_#>uML<|wVNrmMT$^|)8xAI{<=tamy} tw1RHyNQkblf literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00009-dc49acd6-1a95-4e37-bf4b-cfbd789ff11c.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00009-dc49acd6-1a95-4e37-bf4b-cfbd789ff11c.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..fcaf10e376e2e83a8015a74350b2122dd1cd5ac5 GIT binary patch literal 16 XcmYc;N@ieSU}CtZ%>KesY%L=IA%p}d literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00011-f2482f68-663e-4380-bded-b62c066e32c9.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00011-f2482f68-663e-4380-bded-b62c066e32c9.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..af18c5385c62f24566bb8e53f473fd9be983f216 GIT binary patch literal 16 XcmYc;N@ieSU}9L`y5-8t+40{2D-8zG literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00013-fd83bc7c-cb1e-4bf5-8e25-44a270e58690.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00013-fd83bc7c-cb1e-4bf5-8e25-44a270e58690.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..0e1d595b76b9d484384ab79fdb2631853b4c1cb4 GIT binary patch literal 16 XcmYc;N@ieSU}C7weRzVcv^EC-B}xS_ literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00015-78628903-d770-4d3b-984b-365fde6e183a.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/.part-00015-78628903-d770-4d3b-984b-365fde6e183a.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..bec4280c0526b2ff9e5f7d6822d6a649587ced53 GIT binary patch literal 16 XcmYc;N@ieSU}A{wQ!Pol_v<|XBTWV7 literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00001-f38b954d-41f3-41c3-a7a7-3115e28814b7.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00001-f38b954d-41f3-41c3-a7a7-3115e28814b7.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..68aea46efcbe5f131fc5c59001787af811b48523 GIT binary patch literal 1003 zcma)5Pixdb6ras*w<%Oa=nM(uu!MGLVJ9S+WHTv66g-IbPy|5`kx6D!gV{gL?ka96 z^wcll7x3iKPavK=_UISztcM;8g7~ssy9Ez!4w*Oa{oe1r-woR-f)V4H)Q2lljt<})#+Uh+7&e^v%Ujccp;cpuL@$17^?MCXy+QrkWGYLYo@a8fCOk;DIdiaXgB04UCADI@J;GqhYHybeZvlFKc$ZvmR2-w)!Wm(F4|s7t+CYTZZ>e*#Dsil z;7>I~F6%^Z>#%IVoH~XvC=VSm6fhcRHYMkcDo6RCZ>%tfIW97G{y~h63I_&yG*M%x z9K3aUYAmD}3*&w5*dD(dDtp%+jmt?owmWu~t3r@1NK+mAFQdJCd~(vc&V-3AQlUcr|Kt}Z$apkwmiEflTwTj^Jm2wK+sPy^WGjjk sb5ridjPb5YLqBC{Cb}sLl1OEY`5yEAG!Aym4qTJ!raXg#J%DHa2lnXXF#rGn literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00009-dc49acd6-1a95-4e37-bf4b-cfbd789ff11c.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00009-dc49acd6-1a95-4e37-bf4b-cfbd789ff11c.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..3a789a4c626a781f82377079073528c052b590bf GIT binary patch literal 937 zcma)5zl+pB6rRkkOOQnnZiWO>gmAaqL3YUR-7Je>Cx;w|#WpHBo6MfU?2k*biY)sN zSX^VJrC3|}XINNVDcD?TVQJ~hn!8=FFjIs#?|tuk?|t)fdjFA45YdQEzW)6E;kHIF zTw6p&=#GaGLJg?}-|~R}33*Q5|GNAk?ZDWVzp#CP*jC@>UeNExq2+fy;aJ_VANIR} z7xEzR$AK?A^)6Nw`I>acBv_>gZNmGth?H8eidLgCIRSp8YIH;{q|$`Y@IgCY@a*NY zJbEFfoHP+>9$*D0F(Pd+8p|X;Ko2!QQ><#L*^xEURDC43)DlH7#q^k-bmZZ7sI5iO z;mxR9FZ2LnLU|zl)*yTIN+S1AXQYvK#ax!%P`{Pvh13}h>&;pZ1x?jbt{&U&Fjoyi z=|@aHONd@d6nA9$UD^Z9O+}e!XN)g6$jmm1vsWU^lle@!2#af4s-Ow@iIwcz1I`2 t74@yqb^C#cd?#|F2@fLI8~4P-bsgJvoGA2;R1;RN+`3P&B&YC`{sEGT%<=#L literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00011-f2482f68-663e-4380-bded-b62c066e32c9.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00011-f2482f68-663e-4380-bded-b62c066e32c9.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..a46769edeb42628124fe7cf2cd457bb351918620 GIT binary patch literal 936 zcma)5zi-n(6h2=<@em;)^-i|rA+p-2mF!b|mo`5XA+e!?s)`{CAUK~d#lrbf+W}Qk z_70U0Vq|~;R#^Bu7+8=P`(k2oE_HcUl-uJ%u-sg8G_a6rY5r@>t+jqad)f|H1 zx^UD{jTOsU#Ow# zbizo>aA@Dfwq;&(ZkPny6rp8!rxcM@3bvwks7a22AK4BakV~U5A=H0R%ce4Y^*mEA z^+=K`BGp}N;W$F121aETN4w~e189nE$MzbgL#p-xGPmpkMKHzmh#oi0{(7jjMbX~P zsB16u09rzMApKDx+w{sH_fTWtklKp5D7&P-FzBVx81&1{S`GzGl~S%8+vYG=6+-DJ zOuib3{xm3VnDn=?4VrtFHAzpIoJx?HEfr_4b(+PKv2{x@!J5e0{|AwsB@BMf*-Ymw zoxEWqoy$n((t2;q-TZv2-JYA}=}hIW=MH0?M48)rGIz&vRD|LP+$&fz8K1&!JWCQ8 zCJ==*?YY;CZg0L=csH4#{yG&Z#Q#rzwm{a=s#$-O)>qTj0~T<`>wDoWPNI6dtwmFH qnq46}hguzQ1y4gBD$xqtdME@B1m~*T+P6Jexnk=+!;+l98~p?R$k${5 literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00013-fd83bc7c-cb1e-4bf5-8e25-44a270e58690.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-08-15/part-00013-fd83bc7c-cb1e-4bf5-8e25-44a270e58690.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9dce7655b22a6bcf9840fcef6de4a78bcd56a925 GIT binary patch literal 937 zcma)5J!{-R5MAwEjEW-=@GcUl5W+rekQPa+`{XdB3nmzo;+iBRD{0RW(S2CbIfUc7 zR4Lr}9|RY!Qsoz<2q{wp(xmhc$=pTu9i)i1nw@#`X6Ehe>EmY}LBt>?xw*disu~2t zb%&@3Jq$2HsHT9qKHz^rUX$6Z*dBnqxb1o!={cemxlYg)ffEYhJKa&& zi+dxN``u7~gmq25Bi%9y)+s{k@LnY%trD!HHK;}=_V!gC zy^&)<>WI|$v4)cvkp>vGc@poVrv{)Y)(zcmf<$$_h14B=g(8?@dQ4B6>R>a}#-ixp ze$S;K&llE2j8~x~26qWdr;{_dP3CDT z!W5#ACT;7E(dsXjOZz?()Ze5+h2;OqFPBI=E}M16nORO}dd%~gZytp6B#q5ZM{+mn qx;@UjZ5g%vh({yQj(8AuyzSQov;KAfDym{|?-+S-No74M`nn1vUD*XEK?)R<* z6t6p=6SC`3LP$xwjc5L${|P*Yw_iS9-r0ijZ+~I?2x)Ys=&C@}n;srs%k%5qPPSl;%-*Kpt9%X4JGyr>0v_Hd{pBF>@49M%ghtspaE5^1x%vA{( z`$XZZM%Z_a(VEVFYI~@;V;IxqjLVrsnfXd_{z@fjG@Tf?1Q)zPjH7=L>3PiY=bX<~ z#*^u5K316wWhRXe+QQD>%#_`;(=3?>neEu4NX21l_Z~0oi5%ylC`9)f7EdQ`Q zSPo;1Vw!U7Yeu`bST3ELOjLi9iWQ>&CqG>h<9OX{eVkO+(^Z?i>2kMvIGjguSoM7+ t8bPPg6{6EtLCXz9Fp}*+ctc-}gm9a}b%U;VWIDKV`PO~LB{{`M`Um-O&oTf2 literal 0 HcmV?d00001 diff --git a/server/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/src/main/java/io/whitefox/core/types/predicates/BaseOp.java index 3ccdcfab1..74a928bc1 100644 --- a/server/src/main/java/io/whitefox/core/types/predicates/BaseOp.java +++ b/server/src/main/java/io/whitefox/core/types/predicates/BaseOp.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.whitefox.core.types.*; import java.util.List; import java.util.Objects; diff --git a/server/src/main/java/io/whitefox/core/types/predicates/EvalContext.java b/server/src/main/java/io/whitefox/core/types/predicates/EvalContext.java deleted file mode 100644 index 3320d4eee..000000000 --- a/server/src/main/java/io/whitefox/core/types/predicates/EvalContext.java +++ /dev/null @@ -1,17 +0,0 @@ -package io.whitefox.core.types.predicates; - -import org.apache.commons.lang3.tuple.Pair; - -import java.util.Map; - -class EvalContext { - - public EvalContext( - Map partitionValues, Map> statsValues) { - this.partitionValues = partitionValues; - this.statsValues = statsValues; - } - - Map partitionValues; - Map> statsValues; -} \ No newline at end of file diff --git a/server/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java deleted file mode 100644 index 0da4645af..000000000 --- a/server/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ /dev/null @@ -1,105 +0,0 @@ -package io.whitefox.core.types.predicates; - -import io.whitefox.core.types.*; -import java.sql.Date; -import java.sql.Timestamp; -import java.util.List; -import java.util.Objects; - -import io.whitefox.core.types.predicates.LeafOp; -import org.apache.commons.lang3.tuple.Pair; - -public class EvalHelper { - - static Pair, Pair> validateAndGetTypeAndValue( - List children, EvalContext ctx) { - var leftChild = children.get(0); - var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); - var leftVal = leftChild.evalExpectValueAndType(ctx).getLeft(); - - var rightChild = children.get(1); - var rightType = rightChild.evalExpectValueAndType(ctx).getRight(); - var rightVal = rightChild.evalExpectValueAndType(ctx).getLeft(); - // If the types don't match, it implies a malformed predicate tree. - // We simply throw an exception, which will cause filtering to be skipped. - if (!Objects.equals(leftType, rightType)) { - throw new IllegalArgumentException("Type mismatch: " + leftType + " vs " + rightType + " for " - + leftChild + " and " + rightChild); - } - - // We throw an exception for nulls, which will skip filtering. - if (leftVal == null || rightVal == null) { - throw new IllegalArgumentException( - "Comparison with null is not supported: " + leftChild + " and " + rightChild); - } - return Pair.of(Pair.of(leftType, leftVal), Pair.of(rightType, rightVal)); - } - - // Implements "equal" between two leaf operations. - static Boolean equal(List children, EvalContext ctx) { - var typesAndValues = validateAndGetTypeAndValue(children, ctx); - var leftType = typesAndValues.getLeft().getLeft(); - var leftVal = typesAndValues.getLeft().getRight(); - var rightVal = typesAndValues.getRight().getRight(); - - if (BooleanType.BOOLEAN.equals(leftType)) { - return Boolean.valueOf(leftVal) == Boolean.valueOf(rightVal); - } else if (IntegerType.INTEGER.equals(leftType)) { - return Integer.parseInt(leftVal) == Integer.parseInt(rightVal); - } else if (LongType.LONG.equals(leftType)) { - return Long.parseLong(leftVal) == Long.parseLong(rightVal); - } else if (StringType.STRING.equals(leftType)) { - return leftVal.equals(rightVal); - } else if (DateType.DATE.equals(leftType)) { - return Date.valueOf(leftVal).equals(Date.valueOf(rightVal)); - } - throw new IllegalArgumentException("Unsupported type: " + leftType); - } - - // TODO: supported expressions; ie. check if column + constant - // TODO: handle column comparisons with literals - static Boolean lessThan(List children, EvalContext ctx) { - var typesAndValues = validateAndGetTypeAndValue(children, ctx); - var leftType = typesAndValues.getLeft().getLeft(); - var leftVal = typesAndValues.getLeft().getRight(); - var rightVal = typesAndValues.getRight().getRight(); - - if (IntegerType.INTEGER.equals(leftType)) { - return Integer.parseInt(leftVal) < Integer.parseInt(rightVal); - } else if (LongType.LONG.equals(leftType)) { - return Long.parseLong(leftVal) < Long.parseLong(rightVal); - } else if (StringType.STRING.equals(leftType)) { - return leftVal.compareTo(rightVal) < 0; - } else if (DateType.DATE.equals(leftType)) { - return Date.valueOf(leftVal).before(Date.valueOf(rightVal)); - } - throw new IllegalArgumentException("Unsupported type: " + leftType); - } - // Validates that the specified value is in the correct format. - // Throws an exception otherwise. - public static void validateValue(String value, DataType valueType) { - try { - if (BooleanType.BOOLEAN.equals(valueType)) { - Boolean.parseBoolean(value); - } else if (IntegerType.INTEGER.equals(valueType)) { - Integer.parseInt(value); - } else if (LongType.LONG.equals(valueType)) { - Long.parseLong(value); - } else if (DateType.DATE.equals(valueType)) { - Date.valueOf(value); - } else if (FloatType.FLOAT.equals(valueType)) { - Float.parseFloat(value); - } else if (DoubleType.DOUBLE.equals(valueType)) { - Double.parseDouble(value); - // TODO check for non deprecated - } else if (TimestampType.TIMESTAMP.equals(valueType)) { - Timestamp.valueOf(value); - } else { - throw new IllegalArgumentException("Unsupported type: " + valueType); - } - } catch (Exception e) { - throw new IllegalArgumentException( - "Error validating " + value + " for type " + valueType + ": " + e); - } - } -} diff --git a/server/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/src/main/java/io/whitefox/core/types/predicates/LeafOp.java index 64a4c8cd3..26de51e29 100644 --- a/server/src/main/java/io/whitefox/core/types/predicates/LeafOp.java +++ b/server/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -1,6 +1,5 @@ package io.whitefox.core.types.predicates; -import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; diff --git a/server/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java deleted file mode 100644 index ac47252d3..000000000 --- a/server/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java +++ /dev/null @@ -1,259 +0,0 @@ -package io.whitefox.core.types.predicates; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -import java.util.List; -import java.util.stream.Collectors; - -// Represents a non-leaf operation. -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "op" -) -@JsonSubTypes({ - @JsonSubTypes.Type(value = EqualOp.class, name = "equal"), - @JsonSubTypes.Type(value = NotOp.class, name = "not"), - @JsonSubTypes.Type(value = OrOp.class, name = "or"), - @JsonSubTypes.Type(value = IsNullOp.class, name = "null"), - @JsonSubTypes.Type(value = AndOp.class, name = "and"), - @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), - @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), - @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), - @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqualOp") -}) -public abstract class NonLeafOp implements BaseOp { - - @JsonProperty("children") - List children; - - public List getAllChildren() { - // TODO flat map every child - return List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList()); - } -} - -class IsNullOp extends NonLeafOp implements UnaryOp { - - @JsonProperty("children") - List children; - - public IsNullOp(List children) { - this.children = children; - } - - public IsNullOp() { - super(); - } - - @Override - public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); - } - - @Override - public Object eval(EvalContext ctx) { - return ((LeafOp) children.get(0)).isNull(ctx); - } -} - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "equal") -class EqualOp extends NonLeafOp implements BinaryOp { - @JsonProperty("children") - List children; - - public EqualOp() { - super(); - } - - public EqualOp(List children) { - this.children = children; - } - - @Override - public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); - } - - @Override - public Object eval(EvalContext ctx) { - return EvalHelper.equal(children, ctx); - } -} - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "lessThan") -class LessThanOp extends NonLeafOp implements BinaryOp { - @JsonProperty("children") - List children; - - public LessThanOp(List children) { - this.children = children; - } - - public LessThanOp() { - super(); - } - - @Override - public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); - } - - @Override - public Object eval(EvalContext ctx) { - return EvalHelper.lessThan(children, ctx); - } -} - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "lessThanOrEqual") -class LessThanOrEqualOp extends NonLeafOp implements BinaryOp { - - @JsonProperty("children") - List children; - - public LessThanOrEqualOp(List children) { - this.children = children; - } - - public LessThanOrEqualOp() { - super(); - } - - @Override - public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); - } - - @Override - public Object eval(EvalContext ctx) { - return EvalHelper.lessThan(children, ctx) || EvalHelper.equal(children, ctx); - } -} - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "greaterThan") -class GreaterThanOp extends NonLeafOp implements BinaryOp { - - @JsonProperty("children") - List children; - - public GreaterThanOp() { - super(); - } - - @Override - public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); - } - - @Override - public Object eval(EvalContext ctx) { - return !EvalHelper.lessThan(children, ctx) && !EvalHelper.equal(children, ctx); - } -} - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "greaterThanOrEqual") -class GreaterThanOrEqualOp extends NonLeafOp implements BinaryOp { - - @JsonProperty("children") - List children; - - public GreaterThanOrEqualOp() { - super(); - } - - @Override - public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); - } - - @Override - public Object eval(EvalContext ctx) { - return !EvalHelper.lessThan(children, ctx); - } -} - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "and") -class AndOp extends NonLeafOp implements BinaryOp { - - public AndOp(List children) { - this.children = children; - } - - public AndOp() { - super(); - } - - @Override - public void validate() { - validateChildren(children); - } - - @Override - public Object eval(EvalContext ctx) { - return children.stream().allMatch(c -> c.evalExpectBoolean(ctx)); - } -} - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "or") -class OrOp extends NonLeafOp implements BinaryOp { - - public OrOp(List children) { - this.children = children; - } - - public OrOp() { - super(); - } - - @Override - public void validate() { - validateChildren(children); - } - - @Override - public Object eval(EvalContext ctx) { - return children.stream().anyMatch(c -> c.evalExpectBoolean(ctx)); - } -} - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "not") -class NotOp extends NonLeafOp implements UnaryOp { - - @JsonProperty("children") - List children; - - public NotOp(List children) { - this.children = children; - } - - public NotOp() { - super(); - } - - @Override - public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); - } - - @Override - public Object eval(EvalContext ctx) { - return !children.get(0).evalExpectBoolean(ctx); - } -} \ No newline at end of file From fc4d54c2c036b0d9b6437d1606f6105b39f47f89 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 1 Dec 2023 12:55:38 +0100 Subject: [PATCH 04/30] formatting --- .../api/core/JsonPredicatesUtilsTest.java | 40 ++-- .../predicates/PredicateParsingTest.java | 88 ++++---- .../java/io/whitefox/core/ColumnRange.java | 56 +++-- .../main/java/io/whitefox/core/FileStats.java | 83 +++---- .../io/whitefox/core/JsonPredicatesUtils.java | 49 ++--- .../core/services/DeltaSharedTable.java | 11 +- .../core/types/predicates/BaseOp.java | 31 ++- .../predicates/DataTypeDeserializer.java | 62 +++--- .../core/types/predicates/EvalContext.java | 31 ++- .../core/types/predicates/EvalHelper.java | 3 +- .../core/types/predicates/LeafOp.java | 202 +++++++++--------- .../core/types/predicates/NonLeafOp.java | 153 ++++++------- .../core/types/predicates/BaseOp.java | 108 ---------- .../predicates/DataTypeDeserializer.java | 46 ---- .../core/types/predicates/LeafOp.java | 141 ------------ .../types/predicates/LeafOpDeserializer.java | 39 ---- 16 files changed, 389 insertions(+), 754 deletions(-) delete mode 100644 server/src/main/java/io/whitefox/core/types/predicates/BaseOp.java delete mode 100644 server/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java delete mode 100644 server/src/main/java/io/whitefox/core/types/predicates/LeafOp.java delete mode 100644 server/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java diff --git a/server/app/src/test/java/io/whitefox/api/core/JsonPredicatesUtilsTest.java b/server/app/src/test/java/io/whitefox/api/core/JsonPredicatesUtilsTest.java index e5a9cdb5a..91739ea51 100644 --- a/server/app/src/test/java/io/whitefox/api/core/JsonPredicatesUtilsTest.java +++ b/server/app/src/test/java/io/whitefox/api/core/JsonPredicatesUtilsTest.java @@ -1,30 +1,30 @@ package io.whitefox.core; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; +import static io.whitefox.api.server.DeltaTestUtils.deltaTable; +import static io.whitefox.api.server.DeltaTestUtils.deltaTableUri; + import io.delta.standalone.DeltaLog; -import io.whitefox.core.services.DeltaSharedTable; -import io.whitefox.core.types.predicates.BaseOp; +import java.util.stream.Collectors; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Test; -import java.util.stream.Collectors; - -import static io.whitefox.api.server.DeltaTestUtils.deltaTable; -import static io.whitefox.api.server.DeltaTestUtils.deltaTableUri; - public class JsonPredicatesUtilsTest { - @Test - void testCreateEvalContext() { - var PTable = new SharedTable( - "partitioned-delta-table-with-multiple-columns", "default", "share1", deltaTable("partitioned-delta-table-with-multiple-columns")); - - var log = DeltaLog.forTable(new Configuration(), deltaTableUri("partitioned-delta-table-with-multiple-columns")); - var contexts = log.snapshot().getAllFiles().stream().map(JsonPredicatesUtils::createEvalContext).collect(Collectors.toList()); - assert(contexts.size()==2); - var c1 = contexts.get(0); - assert(c1.getPartitionValues().get("date").equals("2021-08-09")); + @Test + void testCreateEvalContext() { + var PTable = new SharedTable( + "partitioned-delta-table-with-multiple-columns", + "default", + "share1", + deltaTable("partitioned-delta-table-with-multiple-columns")); - } + var log = DeltaLog.forTable( + new Configuration(), deltaTableUri("partitioned-delta-table-with-multiple-columns")); + var contexts = log.snapshot().getAllFiles().stream() + .map(JsonPredicatesUtils::createEvalContext) + .collect(Collectors.toList()); + assert (contexts.size() == 2); + var c1 = contexts.get(0); + assert (c1.getPartitionValues().get("date").equals("2021-08-09")); + } } diff --git a/server/app/src/test/java/io/whitefox/api/core/types/predicates/PredicateParsingTest.java b/server/app/src/test/java/io/whitefox/api/core/types/predicates/PredicateParsingTest.java index d741eff0b..2cde7d9a0 100644 --- a/server/app/src/test/java/io/whitefox/api/core/types/predicates/PredicateParsingTest.java +++ b/server/app/src/test/java/io/whitefox/api/core/types/predicates/PredicateParsingTest.java @@ -1,55 +1,49 @@ package io.whitefox.core.types.predicates; +import static io.whitefox.core.services.DeltaSharedTable.parsePredicate; + import com.fasterxml.jackson.core.JsonProcessingException; import org.junit.jupiter.api.Test; -import java.util.Map; - -import static io.whitefox.core.services.DeltaSharedTable.parsePredicate; - public class PredicateParsingTest { - @Test - void testParsingOfEqual() throws JsonProcessingException { - var predicate = "{\n" + - " \"op\": \"equal\",\n" + - " \"children\": [\n" + - " {\"op\": \"column\", \"name\":\"hireDate\", \"valueType\":\"date\"},\n" + - " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + - " ]\n" + - "}"; - var op = parsePredicate(predicate); - op.validate(); - EvalContext evctx = new EvalContext(Map.of("date", "20100312"), Map.of()); - op.eval(evctx); - } - - - @Test - void testParsingOfNested() throws JsonProcessingException { - var predicate = "{\n" + - " \"op\":\"and\",\n" + - " \"children\":[\n" + - " {\n" + - " \"op\":\"equal\",\n" + - " \"children\":[\n" + - " {\"op\":\"column\",\"name\":\"hireDate\",\"valueType\":\"date\"},\n" + - " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + - " ]\n" + - " },\n" + - " {\n" + - " \"op\":\"lessThan\",\"children\":[\n" + - " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + - " {\"op\":\"literal\",\"value\":\"25\",\"valueType\":\"int\"}\n" + - " ]\n" + - " }\n" + - " ]\n" + - "}"; - var op = parsePredicate(predicate); - op.validate(); - EvalContext evctx = new EvalContext(Map.of("hireDate", "20100312"), Map.of()); - op.eval(evctx); - } - - + @Test + void testParsingOfEqual() throws JsonProcessingException { + var predicate = "{\n" + " \"op\": \"equal\",\n" + + " \"children\": [\n" + + " {\"op\": \"column\", \"name\":\"hireDate\", \"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + + " ]\n" + + "}"; + var op = parsePredicate(predicate); + op.validate(); + assert (op instanceof EqualOp); + assert (((EqualOp) op).children.size() == 2); + } + + @Test + void testParsingOfNested() throws JsonProcessingException { + var predicate = "{\n" + " \"op\":\"and\",\n" + + " \"children\":[\n" + + " {\n" + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"hireDate\",\"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"op\":\"lessThan\",\"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + + " {\"op\":\"literal\",\"value\":\"25\",\"valueType\":\"int\"}\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + var op = parsePredicate(predicate); + op.validate(); + assert (op instanceof AndOp); + assert (((AndOp) op).children.size() == 2); + assert (((AndOp) op).children.get(0) instanceof EqualOp); + } } diff --git a/server/core/src/main/java/io/whitefox/core/ColumnRange.java b/server/core/src/main/java/io/whitefox/core/ColumnRange.java index 9487307b0..526ab8fea 100644 --- a/server/core/src/main/java/io/whitefox/core/ColumnRange.java +++ b/server/core/src/main/java/io/whitefox/core/ColumnRange.java @@ -2,33 +2,31 @@ import java.util.Comparator; -public class ColumnRange{ - - T minVal; - T maxVal; - private final Comparator ord; - - public ColumnRange(T minVal, T maxVal, Comparator ord) { - this.minVal = minVal; - this.maxVal = maxVal; - this.ord = ord; - } - - public ColumnRange(T onlyVal, Comparator ord) { - this.minVal = onlyVal; - this.maxVal = onlyVal; - this.ord = ord; - } - - public Boolean contains(T point) { - var c1 = ord.compare(minVal, point); - var c2 = ord.compare(maxVal, point); - return (c1 <= 0 && c2 >= 0); - } - - public static ColumnRange toLong(String minVal, String maxVal) { - return new ColumnRange<>(Long.getLong(minVal), Long.getLong(maxVal), Comparator.naturalOrder()); - } - - +public class ColumnRange { + + T minVal; + T maxVal; + private final Comparator ord; + + public ColumnRange(T minVal, T maxVal, Comparator ord) { + this.minVal = minVal; + this.maxVal = maxVal; + this.ord = ord; + } + + public ColumnRange(T onlyVal, Comparator ord) { + this.minVal = onlyVal; + this.maxVal = onlyVal; + this.ord = ord; + } + + public Boolean contains(T point) { + var c1 = ord.compare(minVal, point); + var c2 = ord.compare(maxVal, point); + return (c1 <= 0 && c2 >= 0); + } + + public static ColumnRange toLong(String minVal, String maxVal) { + return new ColumnRange<>(Long.getLong(minVal), Long.getLong(maxVal), Comparator.naturalOrder()); + } } diff --git a/server/core/src/main/java/io/whitefox/core/FileStats.java b/server/core/src/main/java/io/whitefox/core/FileStats.java index dab50fa7d..2c5b648ae 100644 --- a/server/core/src/main/java/io/whitefox/core/FileStats.java +++ b/server/core/src/main/java/io/whitefox/core/FileStats.java @@ -1,47 +1,50 @@ package io.whitefox.core; import com.fasterxml.jackson.annotation.JsonProperty; - import java.util.Map; public class FileStats { - //{"numRecords":1,"minValues":{"id":0},"maxValues":{"id":0},"nullCount":{"id":0}} - @JsonProperty("numRecords") - String numRecords; - - @JsonProperty("minValues") - Map minValues; - - @JsonProperty("maxValues") - Map maxValues; - - @JsonProperty("nullCount") - Map nullCount; - - public FileStats() { - super(); - } - - public String getNumRecords() { - return numRecords; - } - - public Map getMinValues() { - return minValues; - } - - public Map getMaxValues() { - return maxValues; - } - - public Map getNullCount() { - return nullCount; - } - - public FileStats(String numRecords, Map minValues, Map maxValues, Map nullCount) { - this.numRecords = numRecords; - this.minValues = minValues; - this.maxValues = maxValues; - this.nullCount = nullCount; - } + // {"numRecords":1,"minValues":{"id":0},"maxValues":{"id":0},"nullCount":{"id":0}} + @JsonProperty("numRecords") + String numRecords; + + @JsonProperty("minValues") + Map minValues; + + @JsonProperty("maxValues") + Map maxValues; + + @JsonProperty("nullCount") + Map nullCount; + + public FileStats() { + super(); + } + + public String getNumRecords() { + return numRecords; + } + + public Map getMinValues() { + return minValues; + } + + public Map getMaxValues() { + return maxValues; + } + + public Map getNullCount() { + return nullCount; + } + + public FileStats( + String numRecords, + Map minValues, + Map maxValues, + Map nullCount) { + this.numRecords = numRecords; + this.minValues = minValues; + this.maxValues = maxValues; + this.nullCount = nullCount; + } } diff --git a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java index bfc60ee02..a66e70463 100644 --- a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java +++ b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java @@ -5,37 +5,34 @@ import io.delta.standalone.actions.AddFile; import io.whitefox.core.types.predicates.BaseOp; import io.whitefox.core.types.predicates.EvalContext; -import org.apache.commons.lang3.tuple.Pair; - -import java.util.Comparator; import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; public class JsonPredicatesUtils { - public static BaseOp parsePredicate(String predicate) throws JsonProcessingException { - var mapper = new ObjectMapper(); - return mapper.readValue(predicate, BaseOp.class); - } + public static BaseOp parsePredicate(String predicate) throws JsonProcessingException { + var mapper = new ObjectMapper(); + return mapper.readValue(predicate, BaseOp.class); + } - public static EvalContext createEvalContext(AddFile file){ - var statsString = file.getStats(); - var partitionValues = file.getPartitionValues(); + public static EvalContext createEvalContext(AddFile file) { + var statsString = file.getStats(); + var partitionValues = file.getPartitionValues(); - var mapper = new ObjectMapper(); - try { - var fileStats = mapper.readValue(statsString, FileStats.class); - var maxValues = fileStats.maxValues; - var mappedMinMaxPairs = new java.util.HashMap>(); - fileStats.getMinValues().forEach((minK,minV) -> { - String maxV = maxValues.get(minK); - Pair minMaxPair = Pair.of(minV, maxV); - mappedMinMaxPairs.put(minK, minMaxPair); - }); - return new EvalContext(partitionValues, mappedMinMaxPairs); - } - catch (JsonProcessingException e) { - var message = e.getMessage(); - return new EvalContext(partitionValues, Map.of()); - } + var mapper = new ObjectMapper(); + try { + var fileStats = mapper.readValue(statsString, FileStats.class); + var maxValues = fileStats.maxValues; + var mappedMinMaxPairs = new java.util.HashMap>(); + fileStats.getMinValues().forEach((minK, minV) -> { + String maxV = maxValues.get(minK); + Pair minMaxPair = Pair.of(minV, maxV); + mappedMinMaxPairs.put(minK, minMaxPair); + }); + return new EvalContext(partitionValues, mappedMinMaxPairs); + } catch (JsonProcessingException e) { + var message = e.getMessage(); + return new EvalContext(partitionValues, Map.of()); } + } } diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index 2db756d66..4aa620c15 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -6,6 +6,9 @@ import io.delta.standalone.Snapshot; import io.delta.standalone.actions.AddFile; import io.whitefox.core.*; +import io.whitefox.core.Metadata; +import io.whitefox.core.TableSchema; +import io.whitefox.core.types.predicates.BaseOp; import java.sql.Timestamp; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; @@ -95,13 +98,16 @@ public static BaseOp parsePredicate(String predicate) throws JsonProcessingExcep } public boolean filterFileBasedOnPredicates(List predicates, AddFile f) { + if (predicates == null) { + return true; + } var ctx = JsonPredicatesUtils.createEvalContext(f); return predicates.stream().allMatch(p -> { try { var parsedPredicate = JsonPredicatesUtils.parsePredicate(p); return parsedPredicate.evalExpectBoolean(ctx); } catch (JsonProcessingException e) { - System.out.println("Unable to parse predicate: " + p +" due to: " + e); + System.out.println("Unable to parse predicate: " + p + " due to: " + e); return false; } }); @@ -127,7 +133,8 @@ public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { return new ReadTableResultToBeSigned( new Protocol(Optional.of(1)), metadataFromSnapshot(snapshot), - snapshot.getAllFiles().stream().filter(f -> filterFileBasedOnPredicates(predicates, f)) + snapshot.getAllFiles().stream() + .filter(f -> filterFileBasedOnPredicates(predicates, f)) .map(f -> new TableFileToBeSigned( location() + "/" + f.getPath(), f.getSize(), diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java index 74a928bc1..7aeb2a8c1 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java @@ -2,27 +2,23 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; - +import io.whitefox.core.types.*; import java.util.List; import java.util.Objects; -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "op") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "op") @JsonSubTypes({ - @JsonSubTypes.Type(value = LeafOp.class, names = {"column", "literal"}), -// @JsonSubTypes.Type(value = EqualOp.class, names = {"equal"/*, "not", "or", "and", "lessThan", "lessThanOrEqual", "greaterThan", "greaterThanOrEqualOp" */}), -// @JsonSubTypes.Type(value = ColumnOp.class, name = "column"), -// @JsonSubTypes.Type(value = LiteralOp.class, name = "literal"), - @JsonSubTypes.Type(value = EqualOp.class, name = "equal"), - @JsonSubTypes.Type(value = NotOp.class, name = "not"), - @JsonSubTypes.Type(value = OrOp.class, name = "or"), - @JsonSubTypes.Type(value = AndOp.class, name = "and"), - @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), - @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), - @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), - @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqualOp") + @JsonSubTypes.Type( + value = LeafOp.class, + names = {"column", "literal"}), + @JsonSubTypes.Type(value = EqualOp.class, name = "equal"), + @JsonSubTypes.Type(value = NotOp.class, name = "not"), + @JsonSubTypes.Type(value = OrOp.class, name = "or"), + @JsonSubTypes.Type(value = AndOp.class, name = "and"), + @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), + @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), + @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), + @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqualOp") }) public interface BaseOp { @@ -64,7 +60,6 @@ default Boolean treeDepthExceeds(Integer depth) { } } - // Represents a unary operation. interface UnaryOp { // Validates number of children to be 1. diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java b/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java index 7f560908c..f03af5082 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java @@ -5,42 +5,42 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import io.whitefox.core.types.*; - import java.io.IOException; public class DataTypeDeserializer extends StdDeserializer { - // needed for jackson - public DataTypeDeserializer() { - this(null); - } + // needed for jackson + public DataTypeDeserializer() { + this(null); + } - public DataTypeDeserializer(Class vc) { - super(vc); - } + public DataTypeDeserializer(Class vc) { + super(vc); + } - @Override - public DataType deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { - JsonNode node = jsonParser.getCodec().readTree(jsonParser); - String valueType = node.asText(); + @Override + public DataType deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) + throws IOException { + JsonNode node = jsonParser.getCodec().readTree(jsonParser); + String valueType = node.asText(); - switch (valueType) { - case "date": - return DateType.DATE; - case "int": - return IntegerType.INTEGER; - case "double": - return DoubleType.DOUBLE; - case "float": - return FloatType.FLOAT; - case "string": - return StringType.STRING; - case "timestamp": - return TimestampType.TIMESTAMP; - case "long": - return LongType.LONG; - default: - throw new IOException("Unknown type passed inside a json predicate: " + valueType); - } + switch (valueType) { + case "date": + return DateType.DATE; + case "int": + return IntegerType.INTEGER; + case "double": + return DoubleType.DOUBLE; + case "float": + return FloatType.FLOAT; + case "string": + return StringType.STRING; + case "timestamp": + return TimestampType.TIMESTAMP; + case "long": + return LongType.LONG; + default: + throw new IOException("Unknown type passed inside a json predicate: " + valueType); } -} \ No newline at end of file + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalContext.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalContext.java index a454190ca..e9ad7cd12 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalContext.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalContext.java @@ -1,25 +1,24 @@ package io.whitefox.core.types.predicates; -import org.apache.commons.lang3.tuple.Pair; - import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; public class EvalContext { - public EvalContext( - Map partitionValues, Map> statsValues) { - this.partitionValues = partitionValues; - this.statsValues = statsValues; - } + public EvalContext( + Map partitionValues, Map> statsValues) { + this.partitionValues = partitionValues; + this.statsValues = statsValues; + } - final Map partitionValues; - final Map> statsValues; + final Map partitionValues; + final Map> statsValues; - public Map getPartitionValues() { - return partitionValues; - } + public Map getPartitionValues() { + return partitionValues; + } - public Map> getStatsValues() { - return statsValues; - } -} \ No newline at end of file + public Map> getStatsValues() { + return statsValues; + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index 0f431d17a..c845d8082 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -5,14 +5,13 @@ import java.sql.Timestamp; import java.util.List; import java.util.Objects; - import org.apache.commons.lang3.tuple.Pair; // Only for partition values public class EvalHelper { static Pair, Pair> validateAndGetTypeAndValue( - List children, EvalContext ctx) { + List children, EvalContext ctx) { var leftChild = children.get(0); var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); var leftVal = leftChild.evalExpectValueAndType(ctx).getLeft(); diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java index 26de51e29..694b18317 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -6,136 +6,126 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import io.whitefox.core.types.BooleanType; import io.whitefox.core.types.DataType; -import org.apache.commons.lang3.tuple.Pair; - -//import java.beans.ConstructorProperties; import java.util.List; import java.util.Objects; +import org.apache.commons.lang3.tuple.Pair; -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "op" -) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "op") @JsonSubTypes({ - @JsonSubTypes.Type(value = ColumnOp.class, name = "column"), - @JsonSubTypes.Type(value = LiteralOp.class, name = "literal")} -) -//@JsonDeserialize(using = LeafOpDeserializer.class) + @JsonSubTypes.Type(value = ColumnOp.class, name = "column"), + @JsonSubTypes.Type(value = LiteralOp.class, name = "literal") +}) +// @JsonDeserialize(using = LeafOpDeserializer.class) public abstract class LeafOp implements BaseOp { - abstract Boolean isNull(EvalContext ctx); + abstract Boolean isNull(EvalContext ctx); - @JsonDeserialize(using = DataTypeDeserializer.class) - @JsonProperty("valueType") - DataType valueType; + @JsonDeserialize(using = DataTypeDeserializer.class) + @JsonProperty("valueType") + DataType valueType; - Pair evalExpectValueAndType(EvalContext ctx) { - return (Pair) eval(ctx); - } + Pair evalExpectValueAndType(EvalContext ctx) { + return (Pair) eval(ctx); + } - @Override - public List getAllChildren() { - return List.of(); - } + @Override + public List getAllChildren() { + return List.of(); + } - abstract DataType getOpValueType(); + abstract DataType getOpValueType(); } - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "column") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "column") class ColumnOp extends LeafOp { - @JsonProperty("name") - String name; + @JsonProperty("name") + String name; - public ColumnOp(){ - super(); - } + public ColumnOp() { + super(); + } - public ColumnOp(String name, DataType valueType) { - this.name = name; - this.valueType = valueType; - } + public ColumnOp(String name, DataType valueType) { + this.name = name; + this.valueType = valueType; + } - // Determine if the column value is null. - @Override - public Boolean isNull(EvalContext ctx) { - return resolve(ctx) == null; - } + // Determine if the column value is null. + @Override + public Boolean isNull(EvalContext ctx) { + return resolve(ctx) == null; + } - @Override - public Boolean evalExpectBoolean(EvalContext ctx) { - if (!Objects.equals(valueType, BooleanType.BOOLEAN)) { - throw new IllegalArgumentException("Unsupported type for boolean evaluation: " + valueType); - } - return Boolean.valueOf(resolve(ctx)); + @Override + public Boolean evalExpectBoolean(EvalContext ctx) { + if (!Objects.equals(valueType, BooleanType.BOOLEAN)) { + throw new IllegalArgumentException("Unsupported type for boolean evaluation: " + valueType); } + return Boolean.valueOf(resolve(ctx)); + } - @Override - public DataType getOpValueType() { - return valueType; - } + @Override + public DataType getOpValueType() { + return valueType; + } - @Override - public Object eval(EvalContext ctx) { - return Pair.of(resolve(ctx), valueType); - } + @Override + public Object eval(EvalContext ctx) { + return Pair.of(resolve(ctx), valueType); + } - public void validate() { - if (name == null) { - throw new IllegalArgumentException("Name must be specified: " + this); - } - if (!this.isSupportedType(valueType, false)) { - throw new IllegalArgumentException("Unsupported type: " + valueType); - } + public void validate() { + if (name == null) { + throw new IllegalArgumentException("Name must be specified: " + this); } - - private String resolve(EvalContext ctx) { - return ctx.partitionValues.getOrDefault(name, null); + if (!this.isSupportedType(valueType, false)) { + throw new IllegalArgumentException("Unsupported type: " + valueType); } + } + + private String resolve(EvalContext ctx) { + return ctx.partitionValues.getOrDefault(name, null); + } } -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "literal") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "literal") class LiteralOp extends LeafOp { - @JsonProperty("value") - String value; - - @Override - public void validate() { - if (value == null) { - throw new IllegalArgumentException("Value must be specified: " + this); - } - if (!isSupportedType(valueType, false)) { - throw new IllegalArgumentException("Unsupported type: " + valueType); - } - EvalHelper.validateValue(value, valueType); - } - - @Override - public Object eval(EvalContext ctx) { - return Pair.of(value, valueType); - } - - public LiteralOp() { - super(); - } - - public LiteralOp(String value, DataType valueType) { - this.value = value; - this.valueType = valueType; - } - - @Override - public Boolean isNull(EvalContext ctx) { - return false; - } - - @Override - public DataType getOpValueType() { - return valueType; - } -} \ No newline at end of file + @JsonProperty("value") + String value; + + @Override + public void validate() { + if (value == null) { + throw new IllegalArgumentException("Value must be specified: " + this); + } + if (!isSupportedType(valueType, false)) { + throw new IllegalArgumentException("Unsupported type: " + valueType); + } + EvalHelper.validateValue(value, valueType); + } + + @Override + public Object eval(EvalContext ctx) { + return Pair.of(value, valueType); + } + + public LiteralOp() { + super(); + } + + public LiteralOp(String value, DataType valueType) { + this.value = value; + this.valueType = valueType; + } + + @Override + public Boolean isNull(EvalContext ctx) { + return false; + } + + @Override + public DataType getOpValueType() { + return valueType; + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java index 478e49dee..1d09c37a1 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java @@ -3,35 +3,31 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; - import java.util.List; import java.util.stream.Collectors; // Represents a non-leaf operation. -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "op" -) +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "op") @JsonSubTypes({ - @JsonSubTypes.Type(value = EqualOp.class, name = "equal"), - @JsonSubTypes.Type(value = NotOp.class, name = "not"), - @JsonSubTypes.Type(value = OrOp.class, name = "or"), - @JsonSubTypes.Type(value = IsNullOp.class, name = "null"), - @JsonSubTypes.Type(value = AndOp.class, name = "and"), - @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), - @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), - @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), - @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqualOp") + @JsonSubTypes.Type(value = EqualOp.class, name = "equal"), + @JsonSubTypes.Type(value = NotOp.class, name = "not"), + @JsonSubTypes.Type(value = OrOp.class, name = "or"), + @JsonSubTypes.Type(value = IsNullOp.class, name = "null"), + @JsonSubTypes.Type(value = AndOp.class, name = "and"), + @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), + @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), + @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), + @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqualOp") }) public abstract class NonLeafOp implements BaseOp { - @JsonProperty("children") - List children; + @JsonProperty("children") + List children; - public List getAllChildren() { - // TODO flat map every child - return List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList()); - } + public List getAllChildren() { + // TODO flat map every child + return List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList()); + } } class IsNullOp extends NonLeafOp implements UnaryOp { @@ -48,9 +44,10 @@ public IsNullOp() { } @Override - public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); - } + public void validate() { + validateChildren( + List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } @Override public Object eval(EvalContext ctx) { @@ -58,15 +55,13 @@ public Object eval(EvalContext ctx) { } } -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "equal") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "equal") class EqualOp extends NonLeafOp implements BinaryOp { @JsonProperty("children") List children; public EqualOp() { - super(); + super(); } public EqualOp(List children) { @@ -75,7 +70,8 @@ public EqualOp(List children) { @Override public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + validateChildren( + List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @Override @@ -84,9 +80,7 @@ public Object eval(EvalContext ctx) { } } -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "lessThan") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "lessThan") class LessThanOp extends NonLeafOp implements BinaryOp { @JsonProperty("children") List children; @@ -95,13 +89,14 @@ public LessThanOp(List children) { this.children = children; } - public LessThanOp() { - super(); - } + public LessThanOp() { + super(); + } - @Override + @Override public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + validateChildren( + List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @Override @@ -110,9 +105,7 @@ public Object eval(EvalContext ctx) { } } -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "lessThanOrEqual") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "lessThanOrEqual") class LessThanOrEqualOp extends NonLeafOp implements BinaryOp { @JsonProperty("children") @@ -122,13 +115,14 @@ public LessThanOrEqualOp(List children) { this.children = children; } - public LessThanOrEqualOp() { - super(); - } + public LessThanOrEqualOp() { + super(); + } - @Override + @Override public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + validateChildren( + List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @Override @@ -137,21 +131,20 @@ public Object eval(EvalContext ctx) { } } -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "greaterThan") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "greaterThan") class GreaterThanOp extends NonLeafOp implements BinaryOp { @JsonProperty("children") List children; - public GreaterThanOp() { - super(); - } + public GreaterThanOp() { + super(); + } - @Override + @Override public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + validateChildren( + List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @Override @@ -160,21 +153,20 @@ public Object eval(EvalContext ctx) { } } -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "greaterThanOrEqual") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "greaterThanOrEqual") class GreaterThanOrEqualOp extends NonLeafOp implements BinaryOp { @JsonProperty("children") List children; - public GreaterThanOrEqualOp() { - super(); - } + public GreaterThanOrEqualOp() { + super(); + } - @Override + @Override public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + validateChildren( + List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @Override @@ -183,20 +175,18 @@ public Object eval(EvalContext ctx) { } } -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "and") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "and") class AndOp extends NonLeafOp implements BinaryOp { public AndOp(List children) { this.children = children; } - public AndOp() { - super(); - } + public AndOp() { + super(); + } - @Override + @Override public void validate() { validateChildren(children); } @@ -208,20 +198,18 @@ public Object eval(EvalContext ctx) { } } -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "or") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "or") class OrOp extends NonLeafOp implements BinaryOp { public OrOp(List children) { this.children = children; } - public OrOp() { - super(); - } + public OrOp() { + super(); + } - @Override + @Override public void validate() { validateChildren(children); } @@ -232,9 +220,7 @@ public Object eval(EvalContext ctx) { } } -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "not") +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "not") class NotOp extends NonLeafOp implements UnaryOp { @JsonProperty("children") @@ -244,17 +230,18 @@ public NotOp(List children) { this.children = children; } - public NotOp() { - super(); - } + public NotOp() { + super(); + } - @Override + @Override public void validate() { - validateChildren(List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + validateChildren( + List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @Override public Object eval(EvalContext ctx) { return !children.get(0).evalExpectBoolean(ctx); } -} \ No newline at end of file +} diff --git a/server/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/src/main/java/io/whitefox/core/types/predicates/BaseOp.java deleted file mode 100644 index 74a928bc1..000000000 --- a/server/src/main/java/io/whitefox/core/types/predicates/BaseOp.java +++ /dev/null @@ -1,108 +0,0 @@ -package io.whitefox.core.types.predicates; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -import java.util.List; -import java.util.Objects; - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - include = JsonTypeInfo.As.PROPERTY, - property = "op") -@JsonSubTypes({ - @JsonSubTypes.Type(value = LeafOp.class, names = {"column", "literal"}), -// @JsonSubTypes.Type(value = EqualOp.class, names = {"equal"/*, "not", "or", "and", "lessThan", "lessThanOrEqual", "greaterThan", "greaterThanOrEqualOp" */}), -// @JsonSubTypes.Type(value = ColumnOp.class, name = "column"), -// @JsonSubTypes.Type(value = LiteralOp.class, name = "literal"), - @JsonSubTypes.Type(value = EqualOp.class, name = "equal"), - @JsonSubTypes.Type(value = NotOp.class, name = "not"), - @JsonSubTypes.Type(value = OrOp.class, name = "or"), - @JsonSubTypes.Type(value = AndOp.class, name = "and"), - @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), - @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), - @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), - @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqualOp") -}) -public interface BaseOp { - - void validate(); - - default Boolean isSupportedType(DataType valueType, Boolean forV2) { - if (forV2) { - return (valueType instanceof BooleanType - || valueType instanceof IntegerType - || valueType instanceof StringType - || valueType instanceof DateType - || valueType instanceof LongType - || valueType instanceof TimestampType - || valueType instanceof FloatType - || valueType instanceof DoubleType); - } else { - return (valueType instanceof BooleanType - || valueType instanceof IntegerType - || valueType instanceof StringType - || valueType instanceof DateType - || valueType instanceof LongType); - } - } - - Object eval(EvalContext ctx); - - default Boolean evalExpectBoolean(EvalContext ctx) { - return (Boolean) eval(ctx); - } - - List getAllChildren(); - - default Boolean treeDepthExceeds(Integer depth) { - if (depth <= 0) { - return true; - } else { - return getAllChildren().stream().anyMatch(c -> c.treeDepthExceeds(depth - 1)); - } - } -} - - -// Represents a unary operation. -interface UnaryOp { - // Validates number of children to be 1. - default void validateChildren(List children) { - if (children.size() != 1) - throw new IllegalArgumentException( - this + " : expected 1 but found " + children.size() + " children"); - children.get(0).validate(); - } -} - -interface BinaryOp { - // Validates number of children to be 2. - default void validateChildren(List children) { - if (children.size() != 2) - throw new IllegalArgumentException( - this + " : expected 2 but found " + children.size() + " children"); - children.forEach(BaseOp::validate); - var child1 = children.get(0); - var child2 = children.get(1); - if (child1 instanceof LeafOp && child2 instanceof LeafOp) { - var leftType = ((LeafOp) child1).getOpValueType(); - var rightType = ((LeafOp) child2).getOpValueType(); - if (!Objects.equals(leftType, rightType)) { - throw new IllegalArgumentException("Type mismatch: " + leftType + " vs " + rightType - + " for " + child1 + " and " + child2); - } - } - } -} - -interface NaryOp { - // Validates number of children to be at least 2. - default void validateChildren(List children) { - if (children.size() < 2) { - throw new IllegalArgumentException( - this + " : expected at least 2 but found " + children.size() + " children"); - } - children.forEach(BaseOp::validate); - } -} diff --git a/server/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java b/server/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java deleted file mode 100644 index 7f560908c..000000000 --- a/server/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java +++ /dev/null @@ -1,46 +0,0 @@ -package io.whitefox.core.types.predicates; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import io.whitefox.core.types.*; - -import java.io.IOException; - -public class DataTypeDeserializer extends StdDeserializer { - - // needed for jackson - public DataTypeDeserializer() { - this(null); - } - - public DataTypeDeserializer(Class vc) { - super(vc); - } - - @Override - public DataType deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { - JsonNode node = jsonParser.getCodec().readTree(jsonParser); - String valueType = node.asText(); - - switch (valueType) { - case "date": - return DateType.DATE; - case "int": - return IntegerType.INTEGER; - case "double": - return DoubleType.DOUBLE; - case "float": - return FloatType.FLOAT; - case "string": - return StringType.STRING; - case "timestamp": - return TimestampType.TIMESTAMP; - case "long": - return LongType.LONG; - default: - throw new IOException("Unknown type passed inside a json predicate: " + valueType); - } - } -} \ No newline at end of file diff --git a/server/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/src/main/java/io/whitefox/core/types/predicates/LeafOp.java deleted file mode 100644 index 26de51e29..000000000 --- a/server/src/main/java/io/whitefox/core/types/predicates/LeafOp.java +++ /dev/null @@ -1,141 +0,0 @@ -package io.whitefox.core.types.predicates; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; -import io.whitefox.core.types.BooleanType; -import io.whitefox.core.types.DataType; -import org.apache.commons.lang3.tuple.Pair; - -//import java.beans.ConstructorProperties; -import java.util.List; -import java.util.Objects; - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "op" -) -@JsonSubTypes({ - @JsonSubTypes.Type(value = ColumnOp.class, name = "column"), - @JsonSubTypes.Type(value = LiteralOp.class, name = "literal")} -) -//@JsonDeserialize(using = LeafOpDeserializer.class) -public abstract class LeafOp implements BaseOp { - - abstract Boolean isNull(EvalContext ctx); - - @JsonDeserialize(using = DataTypeDeserializer.class) - @JsonProperty("valueType") - DataType valueType; - - Pair evalExpectValueAndType(EvalContext ctx) { - return (Pair) eval(ctx); - } - - @Override - public List getAllChildren() { - return List.of(); - } - - abstract DataType getOpValueType(); -} - - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "column") -class ColumnOp extends LeafOp { - - @JsonProperty("name") - String name; - - public ColumnOp(){ - super(); - } - - public ColumnOp(String name, DataType valueType) { - this.name = name; - this.valueType = valueType; - } - - // Determine if the column value is null. - @Override - public Boolean isNull(EvalContext ctx) { - return resolve(ctx) == null; - } - - @Override - public Boolean evalExpectBoolean(EvalContext ctx) { - if (!Objects.equals(valueType, BooleanType.BOOLEAN)) { - throw new IllegalArgumentException("Unsupported type for boolean evaluation: " + valueType); - } - return Boolean.valueOf(resolve(ctx)); - } - - @Override - public DataType getOpValueType() { - return valueType; - } - - @Override - public Object eval(EvalContext ctx) { - return Pair.of(resolve(ctx), valueType); - } - - public void validate() { - if (name == null) { - throw new IllegalArgumentException("Name must be specified: " + this); - } - if (!this.isSupportedType(valueType, false)) { - throw new IllegalArgumentException("Unsupported type: " + valueType); - } - } - - private String resolve(EvalContext ctx) { - return ctx.partitionValues.getOrDefault(name, null); - } -} - -@JsonTypeInfo( - use = JsonTypeInfo.Id.NAME, - property = "literal") -class LiteralOp extends LeafOp { - @JsonProperty("value") - String value; - - @Override - public void validate() { - if (value == null) { - throw new IllegalArgumentException("Value must be specified: " + this); - } - if (!isSupportedType(valueType, false)) { - throw new IllegalArgumentException("Unsupported type: " + valueType); - } - EvalHelper.validateValue(value, valueType); - } - - @Override - public Object eval(EvalContext ctx) { - return Pair.of(value, valueType); - } - - public LiteralOp() { - super(); - } - - public LiteralOp(String value, DataType valueType) { - this.value = value; - this.valueType = valueType; - } - - @Override - public Boolean isNull(EvalContext ctx) { - return false; - } - - @Override - public DataType getOpValueType() { - return valueType; - } -} \ No newline at end of file diff --git a/server/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java b/server/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java deleted file mode 100644 index 4e4824e00..000000000 --- a/server/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java +++ /dev/null @@ -1,39 +0,0 @@ -package io.whitefox.core.types.predicates; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import io.whitefox.core.types.DateType; - -import java.io.IOException; - -public class LeafOpDeserializer extends StdDeserializer { - public LeafOpDeserializer() { - this(null); - } - - public LeafOpDeserializer(Class vc) { - super(vc); - } - - @Override - public LeafOp deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { - JsonNode node = jsonParser.getCodec().readTree(jsonParser); - String valueType = node.get("valueType").asText(); - String op = node.get("op").asText(); - - // For example: - switch (op) { - case "column": - String name = node.get("name").asText(); -// switch (valueType) - return new ColumnOp(name, DateType.DATE); - case "literal": - String value = node.get("value").asText(); - return new LiteralOp(value, DateType.DATE); - default: - throw new IOException("type not known"); - } - } -} \ No newline at end of file From e877d728226c193048ee2817ebd10c6a79639b73 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 8 Dec 2023 20:58:09 +0100 Subject: [PATCH 05/30] exception tree --- .../io/whitefox/core/DeltaObjectMapper.java | 26 +++++ .../io/whitefox/core/JsonPredicatesUtils.java | 19 +++- .../core/services/DeltaSharedTable.java | 29 +++--- .../core/types/predicates/BaseOp.java | 98 ++++++++++++------- .../predicates/BasePrimitiveTypeNames.java | 18 ++++ .../predicates/DataTypeDeserializer.java | 23 +++-- .../core/types/predicates/EvalHelper.java | 21 ++-- .../types/predicates/EvaluatorVersion.java | 12 +++ .../core/types/predicates/LeafOp.java | 26 +++-- .../types/predicates/LeafOpDeserializer.java | 39 -------- .../core/types/predicates/NonLeafOp.java | 89 +++++++++++++---- .../types/predicates/NullTypeException.java | 20 ++++ .../types/predicates/PredicateException.java | 7 ++ .../predicates/PredicateParsingException.java | 20 ++++ .../PredicateValidationException.java | 24 +++++ .../predicates/TypeMismatchException.java | 20 ++++ .../predicates/TypeNotSupportedException.java | 16 +++ .../WrongExpectedTypeException.java | 17 ++++ .../core/services/DeltaSharedTableTest.java | 55 +++++++++-- .../core/types}/JsonPredicatesUtilsTest.java | 12 ++- .../predicates/PredicateParsingTest.java | 36 +++++-- 21 files changed, 463 insertions(+), 164 deletions(-) create mode 100644 server/core/src/main/java/io/whitefox/core/DeltaObjectMapper.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/EvaluatorVersion.java delete mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/NullTypeException.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/PredicateException.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/PredicateValidationException.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/TypeMismatchException.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/TypeNotSupportedException.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/WrongExpectedTypeException.java rename server/{app/src/test/java/io/whitefox/api/core => core/src/test/java/io/whitefox/core/types}/JsonPredicatesUtilsTest.java (73%) rename server/{app/src/test/java/io/whitefox/api => core/src/test/java/io/whitefox}/core/types/predicates/PredicateParsingTest.java (53%) diff --git a/server/core/src/main/java/io/whitefox/core/DeltaObjectMapper.java b/server/core/src/main/java/io/whitefox/core/DeltaObjectMapper.java new file mode 100644 index 000000000..8dcaa99e5 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/DeltaObjectMapper.java @@ -0,0 +1,26 @@ +package io.whitefox.core; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import io.whitefox.core.types.DataType; +import io.whitefox.core.types.predicates.DataTypeDeserializer; + + +public class DeltaObjectMapper { + + private static final ObjectMapper objectMapper = newInstance(); + + private static ObjectMapper newInstance() { + ObjectMapper mapper = new ObjectMapper(); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + var customSerializersModule = new SimpleModule(); + customSerializersModule.addDeserializer(DataType.class, new DataTypeDeserializer()); + mapper.registerModule(customSerializersModule); + return mapper; + } + + public static ObjectMapper getInstance() { + return objectMapper; + } +} diff --git a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java index a66e70463..f5793fd33 100644 --- a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java +++ b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java @@ -6,22 +6,29 @@ import io.whitefox.core.types.predicates.BaseOp; import io.whitefox.core.types.predicates.EvalContext; import java.util.Map; + +import io.whitefox.core.types.predicates.PredicateParsingException; import org.apache.commons.lang3.tuple.Pair; public class JsonPredicatesUtils { - public static BaseOp parsePredicate(String predicate) throws JsonProcessingException { - var mapper = new ObjectMapper(); - return mapper.readValue(predicate, BaseOp.class); + private static final ObjectMapper objectMapper = DeltaObjectMapper.getInstance(); + + public static BaseOp parsePredicate(String predicate) throws PredicateParsingException { + try { + return objectMapper.readValue(predicate, BaseOp.class); + } + catch (JsonProcessingException e){ + throw new PredicateParsingException(e); + } } public static EvalContext createEvalContext(AddFile file) { var statsString = file.getStats(); var partitionValues = file.getPartitionValues(); - var mapper = new ObjectMapper(); try { - var fileStats = mapper.readValue(statsString, FileStats.class); + var fileStats = objectMapper.readValue(statsString, FileStats.class); var maxValues = fileStats.maxValues; var mappedMinMaxPairs = new java.util.HashMap>(); fileStats.getMinValues().forEach((minK, minV) -> { @@ -31,6 +38,8 @@ public static EvalContext createEvalContext(AddFile file) { }); return new EvalContext(partitionValues, mappedMinMaxPairs); } catch (JsonProcessingException e) { + + // TODO handle like PredicateParsingException var message = e.getMessage(); return new EvalContext(partitionValues, Map.of()); } diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index 4aa620c15..cd6317363 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -8,15 +8,16 @@ import io.whitefox.core.*; import io.whitefox.core.Metadata; import io.whitefox.core.TableSchema; -import io.whitefox.core.types.predicates.BaseOp; +import io.whitefox.core.types.predicates.PredicateException; +import io.whitefox.core.types.predicates.TypeNotSupportedException; +import jakarta.inject.Inject; + import java.sql.Timestamp; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; -import io.whitefox.core.types.predicates.BaseOp; -import org.apache.hadoop.conf.Configuration; public class DeltaSharedTable implements InternalSharedTable { @@ -87,17 +88,8 @@ public Optional getTableVersion(Optional startingTimestamp) { return getSnapshot(startingTimestamp).map(Snapshot::getVersion); } - public static BaseOp parsePredicate(String predicate) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - try { - return mapper.readValue(predicate, BaseOp.class); - } catch (JsonProcessingException e) { - System.out.println("cant parse predicate"); - throw e; - } - } - - public boolean filterFileBasedOnPredicates(List predicates, AddFile f) { + public boolean filterFilesBasedOnPredicates(List predicates, AddFile f) { + // if there are no predicates return all possible files if (predicates == null) { return true; } @@ -106,9 +98,10 @@ public boolean filterFileBasedOnPredicates(List predicates, AddFile f) { try { var parsedPredicate = JsonPredicatesUtils.parsePredicate(p); return parsedPredicate.evalExpectBoolean(ctx); - } catch (JsonProcessingException e) { - System.out.println("Unable to parse predicate: " + p + " due to: " + e); - return false; + } catch (PredicateException e) { + System.out.println("Caught exception: " + e.getMessage()); + System.out.println("File: " + f.getPath() + " will be used in processing due to failure in parsing or processing the predicate"); + return true; } }); } @@ -134,7 +127,7 @@ public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { new Protocol(Optional.of(1)), metadataFromSnapshot(snapshot), snapshot.getAllFiles().stream() - .filter(f -> filterFileBasedOnPredicates(predicates, f)) + .filter(f -> filterFilesBasedOnPredicates(predicates, f)) .map(f -> new TableFileToBeSigned( location() + "/" + f.getPath(), f.getSize(), diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java index 7aeb2a8c1..3ca73075c 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.whitefox.core.types.*; + import java.util.List; import java.util.Objects; @@ -22,31 +23,34 @@ }) public interface BaseOp { - void validate(); + void validate() throws PredicateException; - default Boolean isSupportedType(DataType valueType, Boolean forV2) { - if (forV2) { - return (valueType instanceof BooleanType - || valueType instanceof IntegerType - || valueType instanceof StringType - || valueType instanceof DateType - || valueType instanceof LongType - || valueType instanceof TimestampType - || valueType instanceof FloatType - || valueType instanceof DoubleType); - } else { + default Boolean isSupportedType(DataType valueType, EvaluatorVersion version) { + if (version == EvaluatorVersion.V2) { return (valueType instanceof BooleanType - || valueType instanceof IntegerType - || valueType instanceof StringType - || valueType instanceof DateType - || valueType instanceof LongType); - } + || valueType instanceof IntegerType + || valueType instanceof StringType + || valueType instanceof DateType + || valueType instanceof LongType + || valueType instanceof TimestampType + || valueType instanceof FloatType + || valueType instanceof DoubleType); + } else return (valueType instanceof BooleanType + || valueType instanceof IntegerType + || valueType instanceof StringType + || valueType instanceof DateType + || valueType instanceof LongType); } - Object eval(EvalContext ctx); + Object eval(EvalContext ctx) throws PredicateException; - default Boolean evalExpectBoolean(EvalContext ctx) { - return (Boolean) eval(ctx); + default Boolean evalExpectBoolean(EvalContext ctx) throws PredicateException { + var res = eval(ctx); + if (res instanceof Boolean) { + return (Boolean) res; + } else { + throw new WrongExpectedTypeException(res, Boolean.class); + } } List getAllChildren(); @@ -60,44 +64,64 @@ default Boolean treeDepthExceeds(Integer depth) { } } +// marker interface for easier exception handling +interface AryOp {}; + // Represents a unary operation. -interface UnaryOp { +interface UnaryOp extends AryOp { // Validates number of children to be 1. - default void validateChildren(List children) { + default void validateChildren(List children) throws TypeNotSupportedException, PredicateValidationException, TypeMismatchException { if (children.size() != 1) - throw new IllegalArgumentException( - this + " : expected 1 but found " + children.size() + " children"); - children.get(0).validate(); + throw new PredicateValidationException(children.size(), this, 1); + try { + children.get(0).validate(); + } catch (PredicateException e) { + throw new RuntimeException(e); + } } } -interface BinaryOp { +interface BinaryOp extends AryOp { // Validates number of children to be 2. - default void validateChildren(List children) { + default void validateChildren(List children) throws TypeMismatchException, PredicateValidationException, TypeNotSupportedException { if (children.size() != 2) - throw new IllegalArgumentException( - this + " : expected 2 but found " + children.size() + " children"); - children.forEach(BaseOp::validate); + throw new PredicateValidationException(children.size(), this, 2); + + // otherwise cannot throw exception in method call of lambda + for (BaseOp c : children) { + try { + c.validate(); + } catch (PredicateException e) { + throw new RuntimeException(e); + } + } + var child1 = children.get(0); var child2 = children.get(1); if (child1 instanceof LeafOp && child2 instanceof LeafOp) { var leftType = ((LeafOp) child1).getOpValueType(); var rightType = ((LeafOp) child2).getOpValueType(); if (!Objects.equals(leftType, rightType)) { - throw new IllegalArgumentException("Type mismatch: " + leftType + " vs " + rightType - + " for " + child1 + " and " + child2); + throw new TypeMismatchException(leftType, rightType); } } } } -interface NaryOp { +// not used currently +interface NaryOp extends AryOp { // Validates number of children to be at least 2. - default void validateChildren(List children) { + default void validateChildren(List children) throws PredicateValidationException, TypeNotSupportedException, TypeMismatchException { if (children.size() < 2) { - throw new IllegalArgumentException( - this + " : expected at least 2 but found " + children.size() + " children"); + throw new PredicateValidationException(children.size(), this, 2); + } + + for (BaseOp c : children) { + try { + c.validate(); + } catch (PredicateException e) { + throw new RuntimeException(e); + } } - children.forEach(BaseOp::validate); } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java new file mode 100644 index 000000000..d2c8002b7 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java @@ -0,0 +1,18 @@ +package io.whitefox.core.types.predicates; + +public enum BasePrimitiveTypeNames { + DATE("date"), + INT("int"), + FLOAT("float"), + DOUBLE("double"), + TIMESTAMP("timestamp"), + LONG("long"), + STRING("string") + ; + + final public String value; + + BasePrimitiveTypeNames(String value) { + this.value = value; + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java b/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java index f03af5082..d71a936bd 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java @@ -1,5 +1,6 @@ package io.whitefox.core.types.predicates; +import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonNode; @@ -7,6 +8,8 @@ import io.whitefox.core.types.*; import java.io.IOException; + + public class DataTypeDeserializer extends StdDeserializer { // needed for jackson @@ -22,25 +25,25 @@ public DataTypeDeserializer(Class vc) { public DataType deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { JsonNode node = jsonParser.getCodec().readTree(jsonParser); - String valueType = node.asText(); + String valueType = node.asText().toUpperCase(); - switch (valueType) { - case "date": + switch (BasePrimitiveTypeNames.valueOf(valueType)) { + case DATE: return DateType.DATE; - case "int": + case INT: return IntegerType.INTEGER; - case "double": + case DOUBLE: return DoubleType.DOUBLE; - case "float": + case FLOAT: return FloatType.FLOAT; - case "string": + case STRING: return StringType.STRING; - case "timestamp": + case TIMESTAMP: return TimestampType.TIMESTAMP; - case "long": + case LONG: return LongType.LONG; default: - throw new IOException("Unknown type passed inside a json predicate: " + valueType); + throw new JsonParseException("Unknown type passed inside a json predicate: " + valueType); } } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index c845d8082..32d3232db 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -5,13 +5,14 @@ import java.sql.Timestamp; import java.util.List; import java.util.Objects; + import org.apache.commons.lang3.tuple.Pair; // Only for partition values public class EvalHelper { static Pair, Pair> validateAndGetTypeAndValue( - List children, EvalContext ctx) { + List children, EvalContext ctx) throws PredicateException { var leftChild = children.get(0); var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); var leftVal = leftChild.evalExpectValueAndType(ctx).getLeft(); @@ -22,20 +23,18 @@ static Pair, Pair> validateAndGetTypeAn // If the types don't match, it implies a malformed predicate tree. // We simply throw an exception, which will cause filtering to be skipped. if (!Objects.equals(leftType, rightType)) { - throw new IllegalArgumentException("Type mismatch: " + leftType + " vs " + rightType + " for " - + leftChild + " and " + rightChild); + throw new TypeMismatchException(leftType, rightType); } // We throw an exception for nulls, which will skip filtering. if (leftVal == null || rightVal == null) { - throw new IllegalArgumentException( - "Comparison with null is not supported: " + leftChild + " and " + rightChild); + throw new NullTypeException(leftChild, rightChild); } return Pair.of(Pair.of(leftType, leftVal), Pair.of(rightType, rightVal)); } // Implements "equal" between two leaf operations. - static Boolean equal(List children, EvalContext ctx) { + static Boolean equal(List children, EvalContext ctx) throws PredicateException { var typesAndValues = validateAndGetTypeAndValue(children, ctx); var leftType = typesAndValues.getLeft().getLeft(); var leftVal = typesAndValues.getLeft().getRight(); @@ -52,12 +51,13 @@ static Boolean equal(List children, EvalContext ctx) { } else if (DateType.DATE.equals(leftType)) { return Date.valueOf(leftVal).equals(Date.valueOf(rightVal)); } - throw new IllegalArgumentException("Unsupported type: " + leftType); + else + throw new TypeNotSupportedException(leftType); } // TODO: supported expressions; ie. check if column + constant // TODO: handle column comparisons with literals - static Boolean lessThan(List children, EvalContext ctx) { + static Boolean lessThan(List children, EvalContext ctx) throws PredicateException { var typesAndValues = validateAndGetTypeAndValue(children, ctx); var leftType = typesAndValues.getLeft().getLeft(); var leftVal = typesAndValues.getLeft().getRight(); @@ -72,7 +72,8 @@ static Boolean lessThan(List children, EvalContext ctx) { } else if (DateType.DATE.equals(leftType)) { return Date.valueOf(leftVal).before(Date.valueOf(rightVal)); } - throw new IllegalArgumentException("Unsupported type: " + leftType); + else + throw new TypeNotSupportedException(leftType); } // Validates that the specified value is in the correct format. // Throws an exception otherwise. @@ -94,7 +95,7 @@ public static void validateValue(String value, DataType valueType) { } else if (TimestampType.TIMESTAMP.equals(valueType)) { Timestamp.valueOf(value); } else { - throw new IllegalArgumentException("Unsupported type: " + valueType); + throw new TypeNotSupportedException(valueType); } } catch (Exception e) { throw new IllegalArgumentException( diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvaluatorVersion.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvaluatorVersion.java new file mode 100644 index 000000000..603e6fb5d --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvaluatorVersion.java @@ -0,0 +1,12 @@ +package io.whitefox.core.types.predicates; + +public enum EvaluatorVersion { + V1("v1"), + V2("v2"); + + public final String value; + + EvaluatorVersion(String value) { + this.value = value; + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java index 694b18317..3343c39ff 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -3,29 +3,33 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import io.whitefox.core.types.BooleanType; import io.whitefox.core.types.DataType; import java.util.List; import java.util.Objects; import org.apache.commons.lang3.tuple.Pair; +import static io.whitefox.core.types.predicates.EvaluatorVersion.V1; + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "op") @JsonSubTypes({ @JsonSubTypes.Type(value = ColumnOp.class, name = "column"), @JsonSubTypes.Type(value = LiteralOp.class, name = "literal") }) -// @JsonDeserialize(using = LeafOpDeserializer.class) public abstract class LeafOp implements BaseOp { abstract Boolean isNull(EvalContext ctx); - @JsonDeserialize(using = DataTypeDeserializer.class) @JsonProperty("valueType") DataType valueType; - Pair evalExpectValueAndType(EvalContext ctx) { - return (Pair) eval(ctx); + Pair evalExpectValueAndType(EvalContext ctx) throws PredicateException { + var res = eval(ctx); + if (res instanceof Pair) { + return (Pair) res; + } else { + throw new WrongExpectedTypeException(res, Pair.class); + } } @Override @@ -72,19 +76,21 @@ public DataType getOpValueType() { @Override public Object eval(EvalContext ctx) { + // TODO: handle case of null column + column ranges return Pair.of(resolve(ctx), valueType); } - public void validate() { + public void validate() throws PredicateException { if (name == null) { throw new IllegalArgumentException("Name must be specified: " + this); } - if (!this.isSupportedType(valueType, false)) { - throw new IllegalArgumentException("Unsupported type: " + valueType); + if (!this.isSupportedType(valueType, V1)) { + throw new TypeNotSupportedException(valueType); } } private String resolve(EvalContext ctx) { + // TODO: handle case of null column + column ranges return ctx.partitionValues.getOrDefault(name, null); } } @@ -95,11 +101,11 @@ class LiteralOp extends LeafOp { String value; @Override - public void validate() { + public void validate() throws PredicateException { if (value == null) { throw new IllegalArgumentException("Value must be specified: " + this); } - if (!isSupportedType(valueType, false)) { + if (!isSupportedType(valueType, V1)) { throw new IllegalArgumentException("Unsupported type: " + valueType); } EvalHelper.validateValue(value, valueType); diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java deleted file mode 100644 index 4e4824e00..000000000 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOpDeserializer.java +++ /dev/null @@ -1,39 +0,0 @@ -package io.whitefox.core.types.predicates; - -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import io.whitefox.core.types.DateType; - -import java.io.IOException; - -public class LeafOpDeserializer extends StdDeserializer { - public LeafOpDeserializer() { - this(null); - } - - public LeafOpDeserializer(Class vc) { - super(vc); - } - - @Override - public LeafOp deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { - JsonNode node = jsonParser.getCodec().readTree(jsonParser); - String valueType = node.get("valueType").asText(); - String op = node.get("op").asText(); - - // For example: - switch (op) { - case "column": - String name = node.get("name").asText(); -// switch (valueType) - return new ColumnOp(name, DateType.DATE); - case "literal": - String value = node.get("value").asText(); - return new LiteralOp(value, DateType.DATE); - default: - throw new IOException("type not known"); - } - } -} \ No newline at end of file diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java index 1d09c37a1..3473aaf2c 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; + import java.util.List; import java.util.stream.Collectors; @@ -44,7 +45,7 @@ public IsNullOp() { } @Override - public void validate() { + public void validate() throws PredicateException { validateChildren( List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @@ -69,13 +70,18 @@ public EqualOp(List children) { } @Override - public void validate() { + public void validate() throws PredicateException { validateChildren( List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @Override - public Object eval(EvalContext ctx) { + public Object eval(EvalContext ctx) throws PredicateException { + try { + this.validate(); + } catch (PredicateException e) { + throw new RuntimeException(e); + } return EvalHelper.equal(children, ctx); } } @@ -94,13 +100,18 @@ public LessThanOp() { } @Override - public void validate() { + public void validate() throws PredicateException { validateChildren( List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @Override - public Object eval(EvalContext ctx) { + public Object eval(EvalContext ctx) throws PredicateException { + try { + this.validate(); + } catch (PredicateException e) { + throw new RuntimeException(e); + } return EvalHelper.lessThan(children, ctx); } } @@ -120,13 +131,18 @@ public LessThanOrEqualOp() { } @Override - public void validate() { + public void validate() throws PredicateException { validateChildren( List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @Override - public Object eval(EvalContext ctx) { + public Object eval(EvalContext ctx) throws PredicateException { + try { + this.validate(); + } catch (PredicateException e) { + throw new RuntimeException(e); + } return EvalHelper.lessThan(children, ctx) || EvalHelper.equal(children, ctx); } } @@ -142,13 +158,18 @@ public GreaterThanOp() { } @Override - public void validate() { + public void validate() throws PredicateException { validateChildren( List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @Override - public Object eval(EvalContext ctx) { + public Object eval(EvalContext ctx) throws PredicateException { + try { + this.validate(); + } catch (PredicateException e) { + throw new RuntimeException(e); + } return !EvalHelper.lessThan(children, ctx) && !EvalHelper.equal(children, ctx); } } @@ -164,13 +185,18 @@ public GreaterThanOrEqualOp() { } @Override - public void validate() { + public void validate() throws PredicateException { validateChildren( List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @Override - public Object eval(EvalContext ctx) { + public Object eval(EvalContext ctx) throws PredicateException { + try { + this.validate(); + } catch (PredicateException e) { + throw new RuntimeException(e); + } return !EvalHelper.lessThan(children, ctx); } } @@ -187,14 +213,24 @@ public AndOp() { } @Override - public void validate() { + public void validate() throws PredicateException { validateChildren(children); } @Override - public Object eval(EvalContext ctx) { + public Object eval(EvalContext ctx) throws PredicateException { + try { + this.validate(); + } catch (PredicateException e) { + throw new RuntimeException(e); + } // short-circuits, so not all exceptions will be thrown - return children.stream().allMatch(c -> c.evalExpectBoolean(ctx)); + for (BaseOp c : children) { + if (!c.evalExpectBoolean(ctx)) { + return false; + } + } + return true; } } @@ -210,13 +246,23 @@ public OrOp() { } @Override - public void validate() { + public void validate() throws PredicateException { validateChildren(children); } @Override - public Object eval(EvalContext ctx) { - return children.stream().anyMatch(c -> c.evalExpectBoolean(ctx)); + public Object eval(EvalContext ctx) throws PredicateException { + try { + this.validate(); + } catch (PredicateException e) { + throw new RuntimeException(e); + } + for (BaseOp c : children) { + if (c.evalExpectBoolean(ctx)) { + return true; + } + } + return false; } } @@ -235,13 +281,18 @@ public NotOp() { } @Override - public void validate() { + public void validate() throws PredicateException { validateChildren( List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); } @Override - public Object eval(EvalContext ctx) { + public Object eval(EvalContext ctx) throws PredicateException { + try { + this.validate(); + } catch (PredicateException e) { + throw new RuntimeException(e); + } return !children.get(0).evalExpectBoolean(ctx); } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NullTypeException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NullTypeException.java new file mode 100644 index 000000000..8d2fb26b6 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NullTypeException.java @@ -0,0 +1,20 @@ +package io.whitefox.core.types.predicates; + +public class NullTypeException extends PredicateException { + + final private BaseOp leftChild; + final private BaseOp rightChild; + + + public NullTypeException(BaseOp leftChild, BaseOp rightChild) { + this.leftChild = leftChild; + this.rightChild = rightChild; + } + + @Override + public String getMessage() { + // TODO: Currently means that the column is not in the partition columns + // Expected is to mean that it is not present at all(use file stats) + return "Comparison with a null value is not supported: " + leftChild.getClass() + " and " + rightChild.getClass(); + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateException.java new file mode 100644 index 000000000..585498dba --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateException.java @@ -0,0 +1,7 @@ +package io.whitefox.core.types.predicates; + + +public class PredicateException extends Exception { + +} + diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java new file mode 100644 index 000000000..3250fce24 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java @@ -0,0 +1,20 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.core.JsonProcessingException; + +import java.util.Arrays; + +public class PredicateParsingException extends PredicateException{ + + private final JsonProcessingException cause; + + public PredicateParsingException(JsonProcessingException cause) { + this.cause = cause; + } + + @Override + public String getMessage() { + return "Parsing of predicate failed due to: " + cause.getMessage() + "\n Stack trace: " + Arrays.toString(cause.getStackTrace()); + } +} + diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateValidationException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateValidationException.java new file mode 100644 index 000000000..e279de086 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateValidationException.java @@ -0,0 +1,24 @@ +package io.whitefox.core.types.predicates; + +public class PredicateValidationException extends PredicateException { + + + final private int actualNumOfChildren; + final private AryOp op; + final private int expectedNumOfChildren; + + public PredicateValidationException(int actualNumOfChildren, AryOp op, int expectedNumOfChildren) { + this.actualNumOfChildren = actualNumOfChildren; + this.op = op; + this.expectedNumOfChildren = expectedNumOfChildren; + } + + + @Override + public String getMessage() { + if (op instanceof NaryOp) + return op + " : expected at least " + expectedNumOfChildren + " children, but found " + actualNumOfChildren + " children"; + else + return op + " : expected " + expectedNumOfChildren + " children, but found " + actualNumOfChildren + " children"; + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/TypeMismatchException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/TypeMismatchException.java new file mode 100644 index 000000000..aab90263a --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/TypeMismatchException.java @@ -0,0 +1,20 @@ +package io.whitefox.core.types.predicates; + +import io.whitefox.core.types.DataType; + +public class TypeMismatchException extends PredicateException { + + final private DataType lType; + final private DataType rType; + + public TypeMismatchException(DataType lType, DataType rType) { + this.lType = lType; + this.rType = rType; + } + + @Override + public String getMessage() { + return "Type are not matching between: " + lType.toString() + "and " + rType; + } + +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/TypeNotSupportedException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/TypeNotSupportedException.java new file mode 100644 index 000000000..59f08e325 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/TypeNotSupportedException.java @@ -0,0 +1,16 @@ +package io.whitefox.core.types.predicates; + +import io.whitefox.core.types.DataType; + +public class TypeNotSupportedException extends PredicateException { + final private DataType type; + + public TypeNotSupportedException(DataType type) { + this.type = type; + } + + @Override + public String getMessage() { + return "Unsupported type: " + type.toString(); + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/WrongExpectedTypeException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/WrongExpectedTypeException.java new file mode 100644 index 000000000..2225a1002 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/WrongExpectedTypeException.java @@ -0,0 +1,17 @@ +package io.whitefox.core.types.predicates; + +public class WrongExpectedTypeException extends PredicateException { + + final private Object evaluationResult; + final private Class expectedType; + + public WrongExpectedTypeException(Object evaluationResult, Class expectedType) { + this.evaluationResult = evaluationResult; + this.expectedType = expectedType; + } + + @Override + public String getMessage() { + return "Evaluation of a Root or Non-Leaf predicate is expected to be of " + expectedType + " type, instead got: " + evaluationResult.getClass(); + } +} diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index f961cdb3c..855abe118 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -5,10 +5,14 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.wildfly.common.Assert.assertTrue; +import io.whitefox.core.Protocol; import io.whitefox.core.ReadTableRequest; import io.whitefox.core.SharedTable; + +import java.sql.Timestamp; import java.time.format.DateTimeParseException; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Test; @@ -64,8 +68,6 @@ void getTableVersionWithFutureTimestamp() throws ExecutionException, Interrupted var version = DTable.getTableVersion(TestDateUtils.parseTimestamp("2024-10-20T10:15:30+01:00")); assertEquals(Optional.empty(), version); } -<<<<<<< HEAD -======= @Test void getTableVersionWithMalformedTimestamp() throws ExecutionException, InterruptedException { @@ -73,18 +75,57 @@ void getTableVersionWithMalformedTimestamp() throws ExecutionException, Interrup var DTable = DeltaSharedTable.of(PTable); assertThrows( DateTimeParseException.class, - () -> DTable.getTableVersion(Optional.of("221rfewdsad10:15:30+01:00"))); + () -> DTable.getTableVersion(Optional.of(Timestamp.valueOf("221rfewdsad10:15:30+01:00")))); } @Test - void queryTable() { + void queryTableWithoutPredicate() { var PTable = new SharedTable( "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of("date = '2021-08-09'"), Optional.empty()); + List.of(), Optional.empty()); + var response = DTable.queryTable(request); + assertEquals(response.protocol(), new Protocol(Optional.of(1))); + assertEquals(response.other().size(), 9); + } + + @Test + void queryTableWithJsonPredicate() { + var predicate = "{" + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"date\",\"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-08-15\",\"valueType\":\"date\"}\n" + + " ]\n" + + "}"; + + var PTable = new SharedTable( + "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); + var DTable = DeltaSharedTable.of(PTable); + var request = new ReadTableRequest.ReadTableCurrentVersion( + List.of(predicate), Optional.empty()); var response = DTable.queryTable(request); - assertTrue(2 == 2); + assertEquals(response.other().size(), 4); } ->>>>>>> 82552ab (wip-first draft of JsonPredicates and a test partitioned table) + + @Test + void queryTableWithInvalidJsonPredicate() { + var predicate = "{" + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"dating\",\"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-08-15\",\"valueType\":\"date\"}\n" + + " ]\n" + + "}"; + + var PTable = new SharedTable( + "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); + var DTable = DeltaSharedTable.of(PTable); + var request = new ReadTableRequest.ReadTableCurrentVersion( + List.of(predicate), Optional.empty()); + var response = DTable.queryTable(request); + assertEquals(response.other().size(), 9); + } + } diff --git a/server/app/src/test/java/io/whitefox/api/core/JsonPredicatesUtilsTest.java b/server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java similarity index 73% rename from server/app/src/test/java/io/whitefox/api/core/JsonPredicatesUtilsTest.java rename to server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java index 91739ea51..2f529d9a7 100644 --- a/server/app/src/test/java/io/whitefox/api/core/JsonPredicatesUtilsTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java @@ -1,10 +1,15 @@ -package io.whitefox.core; +package io.whitefox.core.types; -import static io.whitefox.api.server.DeltaTestUtils.deltaTable; -import static io.whitefox.api.server.DeltaTestUtils.deltaTableUri; +import static io.whitefox.DeltaTestUtils.deltaTable; +import static io.whitefox.DeltaTestUtils.deltaTableUri; +import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.standalone.DeltaLog; import java.util.stream.Collectors; + +import io.whitefox.core.JsonPredicatesUtils; +import io.whitefox.core.SharedTable; +import jakarta.inject.Inject; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Test; @@ -18,6 +23,7 @@ void testCreateEvalContext() { "share1", deltaTable("partitioned-delta-table-with-multiple-columns")); + var log = DeltaLog.forTable( new Configuration(), deltaTableUri("partitioned-delta-table-with-multiple-columns")); var contexts = log.snapshot().getAllFiles().stream() diff --git a/server/app/src/test/java/io/whitefox/api/core/types/predicates/PredicateParsingTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java similarity index 53% rename from server/app/src/test/java/io/whitefox/api/core/types/predicates/PredicateParsingTest.java rename to server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java index 2cde7d9a0..c6dc4b668 100644 --- a/server/app/src/test/java/io/whitefox/api/core/types/predicates/PredicateParsingTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java @@ -1,28 +1,30 @@ package io.whitefox.core.types.predicates; -import static io.whitefox.core.services.DeltaSharedTable.parsePredicate; - import com.fasterxml.jackson.core.JsonProcessingException; +import io.whitefox.core.JsonPredicatesUtils; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertThrows; + public class PredicateParsingTest { @Test - void testParsingOfEqual() throws JsonProcessingException { + void testParsingOfEqual() throws PredicateException { + var predicate = "{\n" + " \"op\": \"equal\",\n" + " \"children\": [\n" + " {\"op\": \"column\", \"name\":\"hireDate\", \"valueType\":\"date\"},\n" + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + " ]\n" + "}"; - var op = parsePredicate(predicate); + var op = JsonPredicatesUtils.parsePredicate(predicate); op.validate(); assert (op instanceof EqualOp); assert (((EqualOp) op).children.size() == 2); } @Test - void testParsingOfNested() throws JsonProcessingException { + void testParsingOfNested() throws PredicateException { var predicate = "{\n" + " \"op\":\"and\",\n" + " \"children\":[\n" + " {\n" @@ -40,10 +42,32 @@ void testParsingOfNested() throws JsonProcessingException { + " }\n" + " ]\n" + "}"; - var op = parsePredicate(predicate); + var op = JsonPredicatesUtils.parsePredicate(predicate); op.validate(); assert (op instanceof AndOp); assert (((AndOp) op).children.size() == 2); assert (((AndOp) op).children.get(0) instanceof EqualOp); } + + @Test + void testCustomExceptionOnBadJson() { + var predicate = "{\n" + " \"op\":\"and\",\n" + + " \"children\":[\n" + + " {\n" + + " \"op\":\"equals\",\n" + + " \"children\":[\n" + + " {\"op\":\"columna\",\"name\":\"hireDate\",\"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"op\":\"lessThans\",\"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + + " {\"op\":\"literal\",\"value\":\"25\",\"valueType\":\"int\"}\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + assertThrows(PredicateParsingException.class , () -> JsonPredicatesUtils.parsePredicate(predicate)); + } } From 038ea31afac1c5727c8a51be3ab2a40a04333a14 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 8 Dec 2023 21:03:06 +0100 Subject: [PATCH 06/30] naming --- .../java/io/whitefox/core/types/predicates/BaseOp.java | 10 +++++----- .../types/predicates/PredicateValidationException.java | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java index 3ca73075c..bf36c3eb1 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java @@ -64,11 +64,11 @@ default Boolean treeDepthExceeds(Integer depth) { } } -// marker interface for easier exception handling -interface AryOp {}; +// marker interface for operator arity used for easier exception handling +interface Arity {}; // Represents a unary operation. -interface UnaryOp extends AryOp { +interface UnaryOp extends Arity { // Validates number of children to be 1. default void validateChildren(List children) throws TypeNotSupportedException, PredicateValidationException, TypeMismatchException { if (children.size() != 1) @@ -81,7 +81,7 @@ default void validateChildren(List children) throws TypeNotSupportedExce } } -interface BinaryOp extends AryOp { +interface BinaryOp extends Arity { // Validates number of children to be 2. default void validateChildren(List children) throws TypeMismatchException, PredicateValidationException, TypeNotSupportedException { if (children.size() != 2) @@ -109,7 +109,7 @@ default void validateChildren(List children) throws TypeMismatchExceptio } // not used currently -interface NaryOp extends AryOp { +interface NaryOp extends Arity { // Validates number of children to be at least 2. default void validateChildren(List children) throws PredicateValidationException, TypeNotSupportedException, TypeMismatchException { if (children.size() < 2) { diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateValidationException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateValidationException.java index e279de086..050a62042 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateValidationException.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateValidationException.java @@ -4,10 +4,10 @@ public class PredicateValidationException extends PredicateException { final private int actualNumOfChildren; - final private AryOp op; + final private Arity op; final private int expectedNumOfChildren; - public PredicateValidationException(int actualNumOfChildren, AryOp op, int expectedNumOfChildren) { + public PredicateValidationException(int actualNumOfChildren, Arity op, int expectedNumOfChildren) { this.actualNumOfChildren = actualNumOfChildren; this.op = op; this.expectedNumOfChildren = expectedNumOfChildren; From 372e500137200831cd97be0726506810238319de Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 15 Dec 2023 11:30:33 +0100 Subject: [PATCH 07/30] wip --- .../java/io/whitefox/core/ColumnRange.java | 15 ++++- .../io/whitefox/core/JsonPredicatesUtils.java | 24 ++++++-- .../core/services/DeltaSharedTable.java | 38 +++++++----- .../core/types/predicates/BaseOp.java | 6 +- .../core/types/predicates/EvalHelper.java | 42 +++++++++++-- .../core/types/predicates/LeafOp.java | 59 ++++++++++++++++++- .../predicates/PredicateParsingException.java | 2 +- .../core/services/DeltaSharedTableTest.java | 20 +++++++ .../core/types/JsonPredicatesUtilsTest.java | 15 +++-- 9 files changed, 187 insertions(+), 34 deletions(-) diff --git a/server/core/src/main/java/io/whitefox/core/ColumnRange.java b/server/core/src/main/java/io/whitefox/core/ColumnRange.java index 526ab8fea..aeaa1c22a 100644 --- a/server/core/src/main/java/io/whitefox/core/ColumnRange.java +++ b/server/core/src/main/java/io/whitefox/core/ColumnRange.java @@ -26,7 +26,20 @@ public Boolean contains(T point) { return (c1 <= 0 && c2 >= 0); } + public Boolean canBeLess(T point) { + return (ord.compare(minVal, point) < 0); + } + + public Boolean canBeGreater(T point) { + return (ord.compare(maxVal, point) > 0); + } + + public static ColumnRange toLong(String minVal, String maxVal) { - return new ColumnRange<>(Long.getLong(minVal), Long.getLong(maxVal), Comparator.naturalOrder()); + return new ColumnRange<>(Long.parseLong(minVal), Long.parseLong(maxVal), Comparator.naturalOrder()); + } + + public static ColumnRange toInt(String minVal, String maxVal) { + return new ColumnRange<>(Integer.parseInt(minVal), Integer.parseInt(maxVal), Comparator.naturalOrder()); } } diff --git a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java index f5793fd33..8021a2ff4 100644 --- a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java +++ b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java @@ -3,6 +3,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.standalone.actions.AddFile; +import io.whitefox.core.types.DataType; +import io.whitefox.core.types.DateType; +import io.whitefox.core.types.IntegerType; import io.whitefox.core.types.predicates.BaseOp; import io.whitefox.core.types.predicates.EvalContext; import java.util.Map; @@ -10,6 +13,8 @@ import io.whitefox.core.types.predicates.PredicateParsingException; import org.apache.commons.lang3.tuple.Pair; +import static io.whitefox.core.types.DateType.DATE; + public class JsonPredicatesUtils { private static final ObjectMapper objectMapper = DeltaObjectMapper.getInstance(); @@ -23,7 +28,18 @@ public static BaseOp parsePredicate(String predicate) throws PredicateParsingExc } } - public static EvalContext createEvalContext(AddFile file) { + public static ColumnRange createColumnRange(String name, EvalContext ctx, DataType valueType) { + var fileStats = ctx.getStatsValues(); + var values = fileStats.get(name); + if (valueType instanceof DateType) { + ColumnRange.toLong(values.getLeft(), values.getRight()); + } + else if (valueType instanceof IntegerType) + return ColumnRange.toInt(values.getLeft(), values.getRight()); + return ColumnRange.toInt(values.getLeft(), values.getRight()); + } + + public static EvalContext createEvalContext(AddFile file) throws PredicateParsingException { var statsString = file.getStats(); var partitionValues = file.getPartitionValues(); @@ -38,10 +54,8 @@ public static EvalContext createEvalContext(AddFile file) { }); return new EvalContext(partitionValues, mappedMinMaxPairs); } catch (JsonProcessingException e) { - - // TODO handle like PredicateParsingException - var message = e.getMessage(); - return new EvalContext(partitionValues, Map.of()); + // should never happen, depends on if the delta implementation changes + throw new PredicateParsingException(e); } } } diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index cd6317363..9a8fc761a 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -1,16 +1,14 @@ package io.whitefox.core.services; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.standalone.DeltaLog; import io.delta.standalone.Snapshot; import io.delta.standalone.actions.AddFile; import io.whitefox.core.*; import io.whitefox.core.Metadata; import io.whitefox.core.TableSchema; +import io.whitefox.core.types.predicates.EvalContext; import io.whitefox.core.types.predicates.PredicateException; -import io.whitefox.core.types.predicates.TypeNotSupportedException; -import jakarta.inject.Inject; +import org.apache.log4j.Logger; import java.sql.Timestamp; import java.time.OffsetDateTime; @@ -19,8 +17,11 @@ import java.util.Optional; import java.util.stream.Collectors; + public class DeltaSharedTable implements InternalSharedTable { + private final Logger logger = Logger.getLogger(this.getClass()); + private final DeltaLog deltaLog; private final TableSchemaConverter tableSchemaConverter; private final SharedTable tableDetails; @@ -88,23 +89,32 @@ public Optional getTableVersion(Optional startingTimestamp) { return getSnapshot(startingTimestamp).map(Snapshot::getVersion); } + + private boolean evaluatePredicate(String predicate, EvalContext ctx, AddFile f) { + try { + var parsedPredicate = JsonPredicatesUtils.parsePredicate(predicate); + return parsedPredicate.evalExpectBoolean(ctx); + } catch (PredicateException e) { + logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage()); + logger.info("File: " + f.getPath() + " will be used in processing due to failure in parsing or processing the predicate: " + predicate); + return true; + } + } + public boolean filterFilesBasedOnPredicates(List predicates, AddFile f) { // if there are no predicates return all possible files if (predicates == null) { return true; } - var ctx = JsonPredicatesUtils.createEvalContext(f); - return predicates.stream().allMatch(p -> { - try { - var parsedPredicate = JsonPredicatesUtils.parsePredicate(p); - return parsedPredicate.evalExpectBoolean(ctx); - } catch (PredicateException e) { - System.out.println("Caught exception: " + e.getMessage()); - System.out.println("File: " + f.getPath() + " will be used in processing due to failure in parsing or processing the predicate"); + try { + var ctx = JsonPredicatesUtils.createEvalContext(f); + return predicates.stream().allMatch(p -> evaluatePredicate(p, ctx, f)); + } catch (PredicateException e) { + logger.debug("Caught exception: " + e.getMessage()); + logger.info("File: " + f.getPath() + " will be used in processing due to failure in parsing or processing the predicate"); return true; } - }); - } + }; public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { List predicates; diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java index bf36c3eb1..1273583da 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java @@ -70,7 +70,7 @@ interface Arity {}; // Represents a unary operation. interface UnaryOp extends Arity { // Validates number of children to be 1. - default void validateChildren(List children) throws TypeNotSupportedException, PredicateValidationException, TypeMismatchException { + default void validateChildren(List children) throws PredicateException { if (children.size() != 1) throw new PredicateValidationException(children.size(), this, 1); try { @@ -83,7 +83,7 @@ default void validateChildren(List children) throws TypeNotSupportedExce interface BinaryOp extends Arity { // Validates number of children to be 2. - default void validateChildren(List children) throws TypeMismatchException, PredicateValidationException, TypeNotSupportedException { + default void validateChildren(List children) throws PredicateException { if (children.size() != 2) throw new PredicateValidationException(children.size(), this, 2); @@ -111,7 +111,7 @@ default void validateChildren(List children) throws TypeMismatchExceptio // not used currently interface NaryOp extends Arity { // Validates number of children to be at least 2. - default void validateChildren(List children) throws PredicateValidationException, TypeNotSupportedException, TypeMismatchException { + default void validateChildren(List children) throws PredicateException { if (children.size() < 2) { throw new PredicateValidationException(children.size(), this, 2); } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index 32d3232db..d983cffb9 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -1,5 +1,6 @@ package io.whitefox.core.types.predicates; +import io.whitefox.core.ColumnRange; import io.whitefox.core.types.*; import java.sql.Date; import java.sql.Timestamp; @@ -11,7 +12,18 @@ // Only for partition values public class EvalHelper { - static Pair, Pair> validateAndGetTypeAndValue( + static Pair validateAndGetRange(List children, EvalContext ctx) throws PredicateException { + var leftChild = (ColumnOp) children.get(0); + var columnRange = leftChild.evalExpectColumnRange(ctx); + + var rightChild = children.get(1); + var rightType = rightChild.evalExpectValueAndType(ctx).getRight(); + var rightVal = rightChild.evalExpectValueAndType(ctx).getLeft(); + + return Pair.of(columnRange, rightVal); + } + + static Pair,Pair, Pair>> validateAndGetTypeAndValue( List children, EvalContext ctx) throws PredicateException { var leftChild = children.get(0); var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); @@ -26,16 +38,29 @@ static Pair, Pair> validateAndGetTypeAn throw new TypeMismatchException(leftType, rightType); } + if (leftVal == null && leftChild instanceof ColumnOp){ + return Pair.of(validateAndGetRange(children, ctx), null); + } + // We throw an exception for nulls, which will skip filtering. if (leftVal == null || rightVal == null) { throw new NullTypeException(leftChild, rightChild); } - return Pair.of(Pair.of(leftType, leftVal), Pair.of(rightType, rightVal)); + return Pair.of(null, Pair.of(Pair.of(leftType, leftVal), Pair.of(rightType, rightVal))); } // Implements "equal" between two leaf operations. static Boolean equal(List children, EvalContext ctx) throws PredicateException { - var typesAndValues = validateAndGetTypeAndValue(children, ctx); + + var columnRangeOrTypeAndValue = validateAndGetTypeAndValue(children, ctx); + if (columnRangeOrTypeAndValue.getLeft() != null) { + var columnRange = columnRangeOrTypeAndValue.getLeft().getLeft(); + var value = columnRangeOrTypeAndValue.getLeft().getRight(); + return columnRange.contains(value); + } + + + var typesAndValues = columnRangeOrTypeAndValue.getRight(); var leftType = typesAndValues.getLeft().getLeft(); var leftVal = typesAndValues.getLeft().getRight(); var rightVal = typesAndValues.getRight().getRight(); @@ -58,7 +83,16 @@ static Boolean equal(List children, EvalContext ctx) throws PredicateExc // TODO: supported expressions; ie. check if column + constant // TODO: handle column comparisons with literals static Boolean lessThan(List children, EvalContext ctx) throws PredicateException { - var typesAndValues = validateAndGetTypeAndValue(children, ctx); + + + var columnRangeOrTypeAndValue = validateAndGetTypeAndValue(children, ctx); + if (columnRangeOrTypeAndValue.getLeft() != null) { + var columnRange = columnRangeOrTypeAndValue.getLeft().getLeft(); + var value = columnRangeOrTypeAndValue.getLeft().getRight(); + return columnRange.contains(value); + } + + var typesAndValues = columnRangeOrTypeAndValue.getRight(); var leftType = typesAndValues.getLeft().getLeft(); var leftVal = typesAndValues.getLeft().getRight(); var rightVal = typesAndValues.getRight().getRight(); diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java index 3343c39ff..0a94f3dec 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -3,12 +3,23 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.delta.standalone.DeltaLog; +import io.delta.standalone.actions.AddFile; +import io.whitefox.core.*; import io.whitefox.core.types.BooleanType; import io.whitefox.core.types.DataType; -import java.util.List; -import java.util.Objects; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Date; +import java.util.*; + +import io.whitefox.core.types.DateType; +import io.whitefox.core.types.IntegerType; import org.apache.commons.lang3.tuple.Pair; +import org.apache.hadoop.conf.Configuration; +import static io.whitefox.core.JsonPredicatesUtils.createColumnRange; import static io.whitefox.core.types.predicates.EvaluatorVersion.V1; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "op") @@ -69,6 +80,10 @@ public Boolean evalExpectBoolean(EvalContext ctx) { return Boolean.valueOf(resolve(ctx)); } + public ColumnRange evalExpectColumnRange(EvalContext ctx) { + return createColumnRange(name, ctx, valueType); + } + @Override public DataType getOpValueType() { return valueType; @@ -89,6 +104,12 @@ public void validate() throws PredicateException { } } + private ColumnRange getColumnRange(EvalContext ctx){ + var fileStats = ctx.getStatsValues(); + var column = fileStats.get(name); + return new ColumnRange<>(column.getLeft(), column.getRight(), Comparator.naturalOrder()); + } + private String resolve(EvalContext ctx) { // TODO: handle case of null column + column ranges return ctx.partitionValues.getOrDefault(name, null); @@ -125,6 +146,40 @@ public LiteralOp(String value, DataType valueType) { this.valueType = valueType; } + private static final Path deltaTablesRoot = Paths.get(".") + .toAbsolutePath() + .resolve("server") + .resolve("core") + .resolve("src/testFixtures/resources/delta/samples") + .toAbsolutePath(); + + public static String deltaTableUri(String tableName) { + return deltaTablesRoot + .resolve(tableName) + .toAbsolutePath() + .normalize() + .toUri() + .toString(); + } + + public static void main(String[] args) throws PredicateException { + var lto = new LessThanOp( + List.of( + new ColumnOp("id", IntegerType.INTEGER), + new LiteralOp("4", IntegerType.INTEGER) + )); + + var log = DeltaLog.forTable( + new Configuration(), deltaTableUri("partitioned-delta-table-with-multiple-columns")); + var contexts = new ArrayList(); + for (AddFile file : log.snapshot().getAllFiles()) { + EvalContext evalContext = JsonPredicatesUtils.createEvalContext(file); + contexts.add(evalContext); + } + + lto.eval(contexts.get(0)); + } + @Override public Boolean isNull(EvalContext ctx) { return false; diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java index 3250fce24..3ca1283f4 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java @@ -4,7 +4,7 @@ import java.util.Arrays; -public class PredicateParsingException extends PredicateException{ +public class PredicateParsingException extends PredicateException { private final JsonProcessingException cause; diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index 855abe118..99c91cabf 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -128,4 +128,24 @@ void queryTableWithInvalidJsonPredicate() { assertEquals(response.other().size(), 9); } + @Test + void queryTableWithColumnRangePredicate() { + var tableName = "partitioned-delta-table-with-multiple-columns"; + var predicate = "{" + + " \"op\":\"lessThan\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + + " {\"op\":\"literal\",\"value\":\"4\",\"valueType\":\"int\"}\n" + + " ]\n" + + "}"; + + var PTable = new SharedTable( + tableName, "default", "share1", deltaTable(tableName)); + var DTable = DeltaSharedTable.of(PTable); + var request = new ReadTableRequest.ReadTableCurrentVersion( + List.of(predicate), Optional.empty()); + var response = DTable.queryTable(request); + assertEquals(response.other().size(), 9); + } + } diff --git a/server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java b/server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java index 2f529d9a7..f0a874186 100644 --- a/server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java @@ -5,10 +5,15 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.standalone.DeltaLog; + +import java.util.ArrayList; +import java.util.List; import java.util.stream.Collectors; +import io.delta.standalone.actions.AddFile; import io.whitefox.core.JsonPredicatesUtils; import io.whitefox.core.SharedTable; +import io.whitefox.core.types.predicates.*; import jakarta.inject.Inject; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Test; @@ -16,7 +21,7 @@ public class JsonPredicatesUtilsTest { @Test - void testCreateEvalContext() { + void testCreateEvalContext() throws PredicateParsingException { var PTable = new SharedTable( "partitioned-delta-table-with-multiple-columns", "default", @@ -26,9 +31,11 @@ void testCreateEvalContext() { var log = DeltaLog.forTable( new Configuration(), deltaTableUri("partitioned-delta-table-with-multiple-columns")); - var contexts = log.snapshot().getAllFiles().stream() - .map(JsonPredicatesUtils::createEvalContext) - .collect(Collectors.toList()); + var contexts = new ArrayList(); + for (AddFile file : log.snapshot().getAllFiles()) { + EvalContext evalContext = JsonPredicatesUtils.createEvalContext(file); + contexts.add(evalContext); + } assert (contexts.size() == 2); var c1 = contexts.get(0); assert (c1.getPartitionValues().get("date").equals("2021-08-09")); From 5d808f31bdb3a8113484912aa5d91bf4da69ae55 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 15 Dec 2023 11:55:31 +0100 Subject: [PATCH 08/30] wip - draft column range evaluation --- .../java/io/whitefox/core/ColumnRange.java | 82 ++++++++++++++----- .../io/whitefox/core/JsonPredicatesUtils.java | 7 +- .../core/types/predicates/LeafOp.java | 40 --------- .../core/services/DeltaSharedTableTest.java | 2 +- 4 files changed, 64 insertions(+), 67 deletions(-) diff --git a/server/core/src/main/java/io/whitefox/core/ColumnRange.java b/server/core/src/main/java/io/whitefox/core/ColumnRange.java index aeaa1c22a..604e89438 100644 --- a/server/core/src/main/java/io/whitefox/core/ColumnRange.java +++ b/server/core/src/main/java/io/whitefox/core/ColumnRange.java @@ -1,45 +1,87 @@ package io.whitefox.core; +import io.whitefox.core.types.DataType; +import io.whitefox.core.types.IntegerType; +import io.whitefox.core.types.LongType; + import java.util.Comparator; -public class ColumnRange { +public class ColumnRange { + + String minVal; + String maxVal; - T minVal; - T maxVal; - private final Comparator ord; + DataType valueType; - public ColumnRange(T minVal, T maxVal, Comparator ord) { + public ColumnRange(String minVal, String maxVal, DataType valueType) { this.minVal = minVal; this.maxVal = maxVal; - this.ord = ord; + this.valueType = valueType; } - public ColumnRange(T onlyVal, Comparator ord) { + public ColumnRange(String onlyVal, DataType valueType) { this.minVal = onlyVal; this.maxVal = onlyVal; - this.ord = ord; + this.valueType = valueType; } - public Boolean contains(T point) { - var c1 = ord.compare(minVal, point); - var c2 = ord.compare(maxVal, point); - return (c1 <= 0 && c2 >= 0); + private Boolean typedContains(String point){ + if (valueType instanceof IntegerType) { + var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point)); + var c2 = Integer.compare(Integer.parseInt(maxVal), Integer.parseInt(point)); + return (c1 <= 0 && c2 >= 0); + } + else if (valueType instanceof LongType){ + var c1 = Long.compare(Long.parseLong(minVal), Long.parseLong(point)); + var c2 = Long.compare(Long.parseLong(maxVal), Long.parseLong(point)); + return (c1 <= 0 && c2 >= 0); + } + else { + var c1 = minVal.compareTo(point); + var c2 = maxVal.compareTo(point); + return (c1 <= 0 && c2 >= 0); + } } - public Boolean canBeLess(T point) { - return (ord.compare(minVal, point) < 0); + private Boolean typedLessThan(String point){ + if (valueType instanceof IntegerType) { + var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point)); + return (c1 < 0); + } + else if (valueType instanceof LongType){ + var c1 = Long.compare(Long.parseLong(minVal), Long.parseLong(point)); + return (c1 < 0); + } + else { + var c = minVal.compareTo(point); + return (c >= 0); + } } - public Boolean canBeGreater(T point) { - return (ord.compare(maxVal, point) > 0); + private Boolean typedGreaterThan(String point){ + if (valueType instanceof IntegerType) { + var c = Integer.compare(Integer.parseInt(maxVal), Integer.parseInt(point)); + return (c > 0); + } + else if (valueType instanceof LongType){ + var c = Long.compare(Long.parseLong(maxVal), Long.parseLong(point)); + return (c > 0); + } + else { + var c = maxVal.compareTo(point); + return (c >= 0); + } } + public Boolean contains(String point) { + return typedContains(point); + } - public static ColumnRange toLong(String minVal, String maxVal) { - return new ColumnRange<>(Long.parseLong(minVal), Long.parseLong(maxVal), Comparator.naturalOrder()); + public Boolean canBeLess(String point) { + return typedLessThan(point); } - public static ColumnRange toInt(String minVal, String maxVal) { - return new ColumnRange<>(Integer.parseInt(minVal), Integer.parseInt(maxVal), Comparator.naturalOrder()); + public Boolean canBeGreater(String point) { + return typedGreaterThan(point); } } diff --git a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java index 8021a2ff4..f2096dbf5 100644 --- a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java +++ b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java @@ -31,12 +31,7 @@ public static BaseOp parsePredicate(String predicate) throws PredicateParsingExc public static ColumnRange createColumnRange(String name, EvalContext ctx, DataType valueType) { var fileStats = ctx.getStatsValues(); var values = fileStats.get(name); - if (valueType instanceof DateType) { - ColumnRange.toLong(values.getLeft(), values.getRight()); - } - else if (valueType instanceof IntegerType) - return ColumnRange.toInt(values.getLeft(), values.getRight()); - return ColumnRange.toInt(values.getLeft(), values.getRight()); + return new ColumnRange(values.getLeft(), values.getRight(), valueType); } public static EvalContext createEvalContext(AddFile file) throws PredicateParsingException { diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java index 0a94f3dec..8fb0d0674 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -104,12 +104,6 @@ public void validate() throws PredicateException { } } - private ColumnRange getColumnRange(EvalContext ctx){ - var fileStats = ctx.getStatsValues(); - var column = fileStats.get(name); - return new ColumnRange<>(column.getLeft(), column.getRight(), Comparator.naturalOrder()); - } - private String resolve(EvalContext ctx) { // TODO: handle case of null column + column ranges return ctx.partitionValues.getOrDefault(name, null); @@ -146,40 +140,6 @@ public LiteralOp(String value, DataType valueType) { this.valueType = valueType; } - private static final Path deltaTablesRoot = Paths.get(".") - .toAbsolutePath() - .resolve("server") - .resolve("core") - .resolve("src/testFixtures/resources/delta/samples") - .toAbsolutePath(); - - public static String deltaTableUri(String tableName) { - return deltaTablesRoot - .resolve(tableName) - .toAbsolutePath() - .normalize() - .toUri() - .toString(); - } - - public static void main(String[] args) throws PredicateException { - var lto = new LessThanOp( - List.of( - new ColumnOp("id", IntegerType.INTEGER), - new LiteralOp("4", IntegerType.INTEGER) - )); - - var log = DeltaLog.forTable( - new Configuration(), deltaTableUri("partitioned-delta-table-with-multiple-columns")); - var contexts = new ArrayList(); - for (AddFile file : log.snapshot().getAllFiles()) { - EvalContext evalContext = JsonPredicatesUtils.createEvalContext(file); - contexts.add(evalContext); - } - - lto.eval(contexts.get(0)); - } - @Override public Boolean isNull(EvalContext ctx) { return false; diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index 99c91cabf..4f0766901 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -145,7 +145,7 @@ void queryTableWithColumnRangePredicate() { var request = new ReadTableRequest.ReadTableCurrentVersion( List.of(predicate), Optional.empty()); var response = DTable.queryTable(request); - assertEquals(response.other().size(), 9); + assertEquals(response.other().size(), 1); } } From f06dc81afafe76f6aa9701afef5d46702b2b77fe Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Sat, 16 Dec 2023 14:35:00 +0100 Subject: [PATCH 09/30] typed eval result --- .../java/io/whitefox/core/ColumnRange.java | 85 +++++++++++-- .../core/types/predicates/EvalHelper.java | 115 ++++++++++-------- .../predicates/LeafEvaluationResult.java | 28 +++++ .../core/types/predicates/LeafOp.java | 16 +-- 4 files changed, 168 insertions(+), 76 deletions(-) create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/LeafEvaluationResult.java diff --git a/server/core/src/main/java/io/whitefox/core/ColumnRange.java b/server/core/src/main/java/io/whitefox/core/ColumnRange.java index 604e89438..a0c7be5ff 100644 --- a/server/core/src/main/java/io/whitefox/core/ColumnRange.java +++ b/server/core/src/main/java/io/whitefox/core/ColumnRange.java @@ -1,10 +1,9 @@ package io.whitefox.core; -import io.whitefox.core.types.DataType; -import io.whitefox.core.types.IntegerType; -import io.whitefox.core.types.LongType; +import io.whitefox.core.types.*; -import java.util.Comparator; +import java.sql.Date; +import java.sql.Timestamp; public class ColumnRange { @@ -36,6 +35,31 @@ else if (valueType instanceof LongType){ var c2 = Long.compare(Long.parseLong(maxVal), Long.parseLong(point)); return (c1 <= 0 && c2 >= 0); } + else if (valueType instanceof TimestampType){ + var c1 = Timestamp.valueOf(minVal).before(Timestamp.valueOf(point)); + var c2 = Timestamp.valueOf(maxVal).before(Timestamp.valueOf(point)); + return c1 && c2; + } + else if (valueType instanceof FloatType){ + var c1 = Float.compare(Float.parseFloat(minVal), Float.parseFloat(point)); + var c2 = Float.compare(Float.parseFloat(maxVal), Float.parseFloat(point)); + return (c1 <= 0 && c2 >= 0); + } + else if (valueType instanceof DoubleType){ + var c1 = Double.compare(Double.parseDouble(minVal), Double.parseDouble(point)); + var c2 = Double.compare(Double.parseDouble(maxVal), Double.parseDouble(point)); + return (c1 <= 0 && c2 >= 0); + } + else if (valueType instanceof DateType){ + var c1 = Date.valueOf(minVal).before(Date.valueOf(point)); + var c2 = Date.valueOf(maxVal).before(Date.valueOf(point)); + return c1 && c2; + } + else if (valueType instanceof BooleanType){ + var c1 = Boolean.parseBoolean(minVal) == Boolean.parseBoolean(point); + var c2 = Boolean.parseBoolean(maxVal) == Boolean.parseBoolean(point); + return c1 || c2; + } else { var c1 = minVal.compareTo(point); var c2 = maxVal.compareTo(point); @@ -43,6 +67,15 @@ else if (valueType instanceof LongType){ } } + public static void main(String[] args) { + var minVal = "4"; + var point = "5"; + var maxVal = "8"; + + var cr = new ColumnRange(minVal,maxVal, IntegerType.INTEGER); + cr.typedLessThan(point); + } + private Boolean typedLessThan(String point){ if (valueType instanceof IntegerType) { var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point)); @@ -52,24 +85,54 @@ else if (valueType instanceof LongType){ var c1 = Long.compare(Long.parseLong(minVal), Long.parseLong(point)); return (c1 < 0); } + else if (valueType instanceof TimestampType){ + return Timestamp.valueOf(minVal).before(Timestamp.valueOf(point)); + } + else if (valueType instanceof FloatType){ + var c1 = Float.compare(Float.parseFloat(minVal), Float.parseFloat(point)); + return (c1 < 0); + } + else if (valueType instanceof DoubleType){ + var c1 = Double.compare(Double.parseDouble(minVal), Double.parseDouble(point)); + return (c1 < 0); + } + else if (valueType instanceof DateType){ + return Date.valueOf(minVal).before(Date.valueOf(point)); + + } else { var c = minVal.compareTo(point); - return (c >= 0); + return (c < 0); } } private Boolean typedGreaterThan(String point){ if (valueType instanceof IntegerType) { - var c = Integer.compare(Integer.parseInt(maxVal), Integer.parseInt(point)); - return (c > 0); + var c = Integer.compare(Integer.parseInt(point), Integer.parseInt(maxVal)); + return (c < 0); } else if (valueType instanceof LongType){ - var c = Long.compare(Long.parseLong(maxVal), Long.parseLong(point)); - return (c > 0); + var c = Long.compare(Long.parseLong(point), Long.parseLong(maxVal)); + return (c < 0); + } + else if (valueType instanceof TimestampType){ + return Timestamp.valueOf(point).before(Timestamp.valueOf(maxVal)); + } + else if (valueType instanceof FloatType){ + var c = Float.compare(Float.parseFloat(maxVal), Float.parseFloat(point)); + return (c < 0); + } + else if (valueType instanceof DoubleType){ + var c = Double.compare(Double.parseDouble(maxVal), Double.parseDouble(point)); + return (c < 0); + } + else if (valueType instanceof DateType){ + return Date.valueOf(point).before(Date.valueOf(maxVal)); + } else { - var c = maxVal.compareTo(point); - return (c >= 0); + var c = point.compareTo(maxVal); + return (c < 0); } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index d983cffb9..227f0a528 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -6,13 +6,14 @@ import java.sql.Timestamp; import java.util.List; import java.util.Objects; +import java.util.Optional; import org.apache.commons.lang3.tuple.Pair; // Only for partition values public class EvalHelper { - static Pair validateAndGetRange(List children, EvalContext ctx) throws PredicateException { + static LeafEvaluationResult validateAndGetRange(List children, EvalContext ctx) throws PredicateException { var leftChild = (ColumnOp) children.get(0); var columnRange = leftChild.evalExpectColumnRange(ctx); @@ -20,10 +21,10 @@ static Pair validateAndGetRange(List children, Eval var rightType = rightChild.evalExpectValueAndType(ctx).getRight(); var rightVal = rightChild.evalExpectValueAndType(ctx).getLeft(); - return Pair.of(columnRange, rightVal); + return LeafEvaluationResult.createFromRange(Pair.of(columnRange, rightVal)); } - static Pair,Pair, Pair>> validateAndGetTypeAndValue( + static LeafEvaluationResult validateAndGetTypeAndValue( List children, EvalContext ctx) throws PredicateException { var leftChild = children.get(0); var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); @@ -39,76 +40,84 @@ static Pair,Pair, Pair children, EvalContext ctx) throws PredicateException { - var columnRangeOrTypeAndValue = validateAndGetTypeAndValue(children, ctx); - if (columnRangeOrTypeAndValue.getLeft() != null) { - var columnRange = columnRangeOrTypeAndValue.getLeft().getLeft(); - var value = columnRangeOrTypeAndValue.getLeft().getRight(); + var leafEvaluationResult = validateAndGetTypeAndValue(children, ctx); + var rangeEvaluation = leafEvaluationResult.rangeEvaluationResult.map(range -> { + var columnRange = range.getLeft(); + var value = range.getRight(); return columnRange.contains(value); + }); + if (rangeEvaluation.isPresent()) + return rangeEvaluation.get(); + else if (leafEvaluationResult.partitionEvaluationResult.isPresent()){ + var typesAndValues = leafEvaluationResult.partitionEvaluationResult.get(); + var leftType = typesAndValues.getLeft().getLeft(); + var leftVal = typesAndValues.getLeft().getRight(); + var rightVal = typesAndValues.getRight().getRight(); + + if (BooleanType.BOOLEAN.equals(leftType)) { + return Boolean.valueOf(leftVal) == Boolean.valueOf(rightVal); + } else if (IntegerType.INTEGER.equals(leftType)) { + return Integer.parseInt(leftVal) == Integer.parseInt(rightVal); + } else if (LongType.LONG.equals(leftType)) { + return Long.parseLong(leftVal) == Long.parseLong(rightVal); + } else if (StringType.STRING.equals(leftType)) { + return leftVal.equals(rightVal); + } else if (DateType.DATE.equals(leftType)) { + return Date.valueOf(leftVal).equals(Date.valueOf(rightVal)); + } + else + throw new TypeNotSupportedException(leftType); } - - - var typesAndValues = columnRangeOrTypeAndValue.getRight(); - var leftType = typesAndValues.getLeft().getLeft(); - var leftVal = typesAndValues.getLeft().getRight(); - var rightVal = typesAndValues.getRight().getRight(); - - if (BooleanType.BOOLEAN.equals(leftType)) { - return Boolean.valueOf(leftVal) == Boolean.valueOf(rightVal); - } else if (IntegerType.INTEGER.equals(leftType)) { - return Integer.parseInt(leftVal) == Integer.parseInt(rightVal); - } else if (LongType.LONG.equals(leftType)) { - return Long.parseLong(leftVal) == Long.parseLong(rightVal); - } else if (StringType.STRING.equals(leftType)) { - return leftVal.equals(rightVal); - } else if (DateType.DATE.equals(leftType)) { - return Date.valueOf(leftVal).equals(Date.valueOf(rightVal)); - } - else - throw new TypeNotSupportedException(leftType); + else + throw new PredicateException(); } - // TODO: supported expressions; ie. check if column + constant - // TODO: handle column comparisons with literals static Boolean lessThan(List children, EvalContext ctx) throws PredicateException { - - var columnRangeOrTypeAndValue = validateAndGetTypeAndValue(children, ctx); - if (columnRangeOrTypeAndValue.getLeft() != null) { - var columnRange = columnRangeOrTypeAndValue.getLeft().getLeft(); - var value = columnRangeOrTypeAndValue.getLeft().getRight(); - return columnRange.contains(value); - } - - var typesAndValues = columnRangeOrTypeAndValue.getRight(); - var leftType = typesAndValues.getLeft().getLeft(); - var leftVal = typesAndValues.getLeft().getRight(); - var rightVal = typesAndValues.getRight().getRight(); - - if (IntegerType.INTEGER.equals(leftType)) { - return Integer.parseInt(leftVal) < Integer.parseInt(rightVal); - } else if (LongType.LONG.equals(leftType)) { - return Long.parseLong(leftVal) < Long.parseLong(rightVal); - } else if (StringType.STRING.equals(leftType)) { - return leftVal.compareTo(rightVal) < 0; - } else if (DateType.DATE.equals(leftType)) { - return Date.valueOf(leftVal).before(Date.valueOf(rightVal)); + var leafEvaluationResult = validateAndGetTypeAndValue(children, ctx); + var rangeEvaluation = leafEvaluationResult.rangeEvaluationResult.map(range -> { + var columnRange = range.getLeft(); + var value = range.getRight(); + return columnRange.canBeLess(value); + }); + + if (rangeEvaluation.isPresent()) + return rangeEvaluation.get(); + else if (leafEvaluationResult.partitionEvaluationResult.isPresent()){ + var typesAndValues = leafEvaluationResult.partitionEvaluationResult.get(); + var leftType = typesAndValues.getLeft().getLeft(); + var leftVal = typesAndValues.getLeft().getRight(); + var rightVal = typesAndValues.getRight().getRight(); + + if (IntegerType.INTEGER.equals(leftType)) { + return Integer.parseInt(leftVal) < Integer.parseInt(rightVal); + } else if (LongType.LONG.equals(leftType)) { + return Long.parseLong(leftVal) < Long.parseLong(rightVal); + } else if (StringType.STRING.equals(leftType)) { + return leftVal.compareTo(rightVal) < 0; + } else if (DateType.DATE.equals(leftType)) { + return Date.valueOf(leftVal).before(Date.valueOf(rightVal)); + } + else + throw new TypeNotSupportedException(leftType); } else - throw new TypeNotSupportedException(leftType); + throw new PredicateException(); } + // Validates that the specified value is in the correct format. // Throws an exception otherwise. public static void validateValue(String value, DataType valueType) { diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafEvaluationResult.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafEvaluationResult.java new file mode 100644 index 000000000..047db20b4 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafEvaluationResult.java @@ -0,0 +1,28 @@ +package io.whitefox.core.types.predicates; + +import io.whitefox.core.ColumnRange; +import io.whitefox.core.types.DataType; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.Optional; + +public class LeafEvaluationResult { + + Optional> rangeEvaluationResult; + Optional, Pair>> partitionEvaluationResult; + + public LeafEvaluationResult(Optional> rangeEvaluationResult, Optional, Pair>> partitionEvaluationResult) { + this.rangeEvaluationResult = rangeEvaluationResult; + this.partitionEvaluationResult = partitionEvaluationResult; + } + + public static LeafEvaluationResult createFromRange(Pair rangeEvaluationResult) { + return new LeafEvaluationResult(Optional.of(rangeEvaluationResult), Optional.empty()); + } + + public static LeafEvaluationResult createFromPartitionColumn(Pair, Pair> partitionEvaluationResult) { + return new LeafEvaluationResult(Optional.empty(), Optional.of(partitionEvaluationResult)); + } + + +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java index 8fb0d0674..71b36442e 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -3,21 +3,13 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.delta.standalone.DeltaLog; -import io.delta.standalone.actions.AddFile; -import io.whitefox.core.*; +import io.whitefox.core.ColumnRange; import io.whitefox.core.types.BooleanType; import io.whitefox.core.types.DataType; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.sql.Date; -import java.util.*; - -import io.whitefox.core.types.DateType; -import io.whitefox.core.types.IntegerType; import org.apache.commons.lang3.tuple.Pair; -import org.apache.hadoop.conf.Configuration; + +import java.util.List; +import java.util.Objects; import static io.whitefox.core.JsonPredicatesUtils.createColumnRange; import static io.whitefox.core.types.predicates.EvaluatorVersion.V1; From 05493246be0933e8f8df7899337c58f26f71f701 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Sat, 16 Dec 2023 15:22:10 +0100 Subject: [PATCH 10/30] Handle non-existing columns in predicate; formatting --- .../java/io/whitefox/core/ColumnRange.java | 66 +++++++------------ .../io/whitefox/core/DeltaObjectMapper.java | 25 ++++--- .../io/whitefox/core/JsonPredicatesUtils.java | 17 ++--- .../core/services/DeltaSharedTable.java | 36 +++++----- .../core/types/predicates/BaseOp.java | 35 +++++----- .../predicates/BasePrimitiveTypeNames.java | 23 ++++--- .../predicates/DataTypeDeserializer.java | 2 - .../core/types/predicates/EvalHelper.java | 57 +++++++--------- .../types/predicates/EvaluatorVersion.java | 12 ++-- .../predicates/LeafEvaluationResult.java | 39 +++++------ .../core/types/predicates/LeafOp.java | 11 ++-- .../NonExistingColumnException.java | 14 ++++ .../core/types/predicates/NonLeafOp.java | 1 - .../types/predicates/NullTypeException.java | 26 ++++---- .../types/predicates/PredicateException.java | 6 +- .../predicates/PredicateParsingException.java | 19 +++--- .../PredicateValidationException.java | 35 +++++----- .../predicates/TypeMismatchException.java | 21 +++--- .../predicates/TypeNotSupportedException.java | 16 ++--- .../WrongExpectedTypeException.java | 21 +++--- .../core/services/DeltaSharedTableTest.java | 62 ++++++++--------- .../core/types/JsonPredicatesUtilsTest.java | 9 +-- .../predicates/PredicateParsingTest.java | 40 +++++------ 23 files changed, 278 insertions(+), 315 deletions(-) create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/NonExistingColumnException.java diff --git a/server/core/src/main/java/io/whitefox/core/ColumnRange.java b/server/core/src/main/java/io/whitefox/core/ColumnRange.java index a0c7be5ff..3efa1d1c3 100644 --- a/server/core/src/main/java/io/whitefox/core/ColumnRange.java +++ b/server/core/src/main/java/io/whitefox/core/ColumnRange.java @@ -1,7 +1,6 @@ package io.whitefox.core; import io.whitefox.core.types.*; - import java.sql.Date; import java.sql.Timestamp; @@ -24,43 +23,36 @@ public ColumnRange(String onlyVal, DataType valueType) { this.valueType = valueType; } - private Boolean typedContains(String point){ + private Boolean typedContains(String point) { if (valueType instanceof IntegerType) { var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point)); var c2 = Integer.compare(Integer.parseInt(maxVal), Integer.parseInt(point)); return (c1 <= 0 && c2 >= 0); - } - else if (valueType instanceof LongType){ + } else if (valueType instanceof LongType) { var c1 = Long.compare(Long.parseLong(minVal), Long.parseLong(point)); var c2 = Long.compare(Long.parseLong(maxVal), Long.parseLong(point)); return (c1 <= 0 && c2 >= 0); - } - else if (valueType instanceof TimestampType){ + } else if (valueType instanceof TimestampType) { var c1 = Timestamp.valueOf(minVal).before(Timestamp.valueOf(point)); var c2 = Timestamp.valueOf(maxVal).before(Timestamp.valueOf(point)); return c1 && c2; - } - else if (valueType instanceof FloatType){ + } else if (valueType instanceof FloatType) { var c1 = Float.compare(Float.parseFloat(minVal), Float.parseFloat(point)); var c2 = Float.compare(Float.parseFloat(maxVal), Float.parseFloat(point)); return (c1 <= 0 && c2 >= 0); - } - else if (valueType instanceof DoubleType){ + } else if (valueType instanceof DoubleType) { var c1 = Double.compare(Double.parseDouble(minVal), Double.parseDouble(point)); var c2 = Double.compare(Double.parseDouble(maxVal), Double.parseDouble(point)); return (c1 <= 0 && c2 >= 0); - } - else if (valueType instanceof DateType){ + } else if (valueType instanceof DateType) { var c1 = Date.valueOf(minVal).before(Date.valueOf(point)); var c2 = Date.valueOf(maxVal).before(Date.valueOf(point)); return c1 && c2; - } - else if (valueType instanceof BooleanType){ + } else if (valueType instanceof BooleanType) { var c1 = Boolean.parseBoolean(minVal) == Boolean.parseBoolean(point); var c2 = Boolean.parseBoolean(maxVal) == Boolean.parseBoolean(point); return c1 || c2; - } - else { + } else { var c1 = minVal.compareTo(point); var c2 = maxVal.compareTo(point); return (c1 <= 0 && c2 >= 0); @@ -72,65 +64,53 @@ public static void main(String[] args) { var point = "5"; var maxVal = "8"; - var cr = new ColumnRange(minVal,maxVal, IntegerType.INTEGER); + var cr = new ColumnRange(minVal, maxVal, IntegerType.INTEGER); cr.typedLessThan(point); } - private Boolean typedLessThan(String point){ + private Boolean typedLessThan(String point) { if (valueType instanceof IntegerType) { var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point)); return (c1 < 0); - } - else if (valueType instanceof LongType){ + } else if (valueType instanceof LongType) { var c1 = Long.compare(Long.parseLong(minVal), Long.parseLong(point)); return (c1 < 0); - } - else if (valueType instanceof TimestampType){ + } else if (valueType instanceof TimestampType) { return Timestamp.valueOf(minVal).before(Timestamp.valueOf(point)); - } - else if (valueType instanceof FloatType){ + } else if (valueType instanceof FloatType) { var c1 = Float.compare(Float.parseFloat(minVal), Float.parseFloat(point)); return (c1 < 0); - } - else if (valueType instanceof DoubleType){ + } else if (valueType instanceof DoubleType) { var c1 = Double.compare(Double.parseDouble(minVal), Double.parseDouble(point)); return (c1 < 0); - } - else if (valueType instanceof DateType){ + } else if (valueType instanceof DateType) { return Date.valueOf(minVal).before(Date.valueOf(point)); - } - else { + } else { var c = minVal.compareTo(point); return (c < 0); } } - private Boolean typedGreaterThan(String point){ + private Boolean typedGreaterThan(String point) { if (valueType instanceof IntegerType) { var c = Integer.compare(Integer.parseInt(point), Integer.parseInt(maxVal)); return (c < 0); - } - else if (valueType instanceof LongType){ + } else if (valueType instanceof LongType) { var c = Long.compare(Long.parseLong(point), Long.parseLong(maxVal)); return (c < 0); - } - else if (valueType instanceof TimestampType){ + } else if (valueType instanceof TimestampType) { return Timestamp.valueOf(point).before(Timestamp.valueOf(maxVal)); - } - else if (valueType instanceof FloatType){ + } else if (valueType instanceof FloatType) { var c = Float.compare(Float.parseFloat(maxVal), Float.parseFloat(point)); return (c < 0); - } - else if (valueType instanceof DoubleType){ + } else if (valueType instanceof DoubleType) { var c = Double.compare(Double.parseDouble(maxVal), Double.parseDouble(point)); return (c < 0); - } - else if (valueType instanceof DateType){ + } else if (valueType instanceof DateType) { return Date.valueOf(point).before(Date.valueOf(maxVal)); - } - else { + } else { var c = point.compareTo(maxVal); return (c < 0); } diff --git a/server/core/src/main/java/io/whitefox/core/DeltaObjectMapper.java b/server/core/src/main/java/io/whitefox/core/DeltaObjectMapper.java index 8dcaa99e5..12d00580c 100644 --- a/server/core/src/main/java/io/whitefox/core/DeltaObjectMapper.java +++ b/server/core/src/main/java/io/whitefox/core/DeltaObjectMapper.java @@ -6,21 +6,20 @@ import io.whitefox.core.types.DataType; import io.whitefox.core.types.predicates.DataTypeDeserializer; - public class DeltaObjectMapper { - private static final ObjectMapper objectMapper = newInstance(); + private static final ObjectMapper objectMapper = newInstance(); - private static ObjectMapper newInstance() { - ObjectMapper mapper = new ObjectMapper(); - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); - var customSerializersModule = new SimpleModule(); - customSerializersModule.addDeserializer(DataType.class, new DataTypeDeserializer()); - mapper.registerModule(customSerializersModule); - return mapper; - } + private static ObjectMapper newInstance() { + ObjectMapper mapper = new ObjectMapper(); + mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); + var customSerializersModule = new SimpleModule(); + customSerializersModule.addDeserializer(DataType.class, new DataTypeDeserializer()); + mapper.registerModule(customSerializersModule); + return mapper; + } - public static ObjectMapper getInstance() { - return objectMapper; - } + public static ObjectMapper getInstance() { + return objectMapper; + } } diff --git a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java index f2096dbf5..67d8ac4db 100644 --- a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java +++ b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java @@ -4,17 +4,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.standalone.actions.AddFile; import io.whitefox.core.types.DataType; -import io.whitefox.core.types.DateType; -import io.whitefox.core.types.IntegerType; import io.whitefox.core.types.predicates.BaseOp; import io.whitefox.core.types.predicates.EvalContext; -import java.util.Map; - +import io.whitefox.core.types.predicates.NonExistingColumnException; import io.whitefox.core.types.predicates.PredicateParsingException; +import java.util.Optional; import org.apache.commons.lang3.tuple.Pair; -import static io.whitefox.core.types.DateType.DATE; - public class JsonPredicatesUtils { private static final ObjectMapper objectMapper = DeltaObjectMapper.getInstance(); @@ -22,15 +18,16 @@ public class JsonPredicatesUtils { public static BaseOp parsePredicate(String predicate) throws PredicateParsingException { try { return objectMapper.readValue(predicate, BaseOp.class); - } - catch (JsonProcessingException e){ + } catch (JsonProcessingException e) { throw new PredicateParsingException(e); } } - public static ColumnRange createColumnRange(String name, EvalContext ctx, DataType valueType) { + public static ColumnRange createColumnRange(String name, EvalContext ctx, DataType valueType) + throws NonExistingColumnException { var fileStats = ctx.getStatsValues(); - var values = fileStats.get(name); + var values = Optional.ofNullable(fileStats.get(name)) + .orElseThrow(() -> new NonExistingColumnException(name)); return new ColumnRange(values.getLeft(), values.getRight(), valueType); } diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index 9a8fc761a..81198a125 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -8,14 +8,13 @@ import io.whitefox.core.TableSchema; import io.whitefox.core.types.predicates.EvalContext; import io.whitefox.core.types.predicates.PredicateException; -import org.apache.log4j.Logger; - import java.sql.Timestamp; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import org.apache.log4j.Logger; public class DeltaSharedTable implements InternalSharedTable { @@ -89,17 +88,18 @@ public Optional getTableVersion(Optional startingTimestamp) { return getSnapshot(startingTimestamp).map(Snapshot::getVersion); } - - private boolean evaluatePredicate(String predicate, EvalContext ctx, AddFile f) { - try { - var parsedPredicate = JsonPredicatesUtils.parsePredicate(predicate); - return parsedPredicate.evalExpectBoolean(ctx); - } catch (PredicateException e) { - logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage()); - logger.info("File: " + f.getPath() + " will be used in processing due to failure in parsing or processing the predicate: " + predicate); - return true; - } + private boolean evaluatePredicate(String predicate, EvalContext ctx, AddFile f) { + try { + var parsedPredicate = JsonPredicatesUtils.parsePredicate(predicate); + return parsedPredicate.evalExpectBoolean(ctx); + } catch (PredicateException e) { + logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage()); + logger.info("File: " + f.getPath() + + " will be used in processing due to failure in parsing or processing the predicate: " + + predicate); + return true; } + } public boolean filterFilesBasedOnPredicates(List predicates, AddFile f) { // if there are no predicates return all possible files @@ -110,11 +110,13 @@ public boolean filterFilesBasedOnPredicates(List predicates, AddFile f) var ctx = JsonPredicatesUtils.createEvalContext(f); return predicates.stream().allMatch(p -> evaluatePredicate(p, ctx, f)); } catch (PredicateException e) { - logger.debug("Caught exception: " + e.getMessage()); - logger.info("File: " + f.getPath() + " will be used in processing due to failure in parsing or processing the predicate"); - return true; - } - }; + logger.debug("Caught exception: " + e.getMessage()); + logger.info("File: " + f.getPath() + + " will be used in processing due to failure in parsing or processing the predicate"); + return true; + } + } + ; public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { List predicates; diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java index 1273583da..f229ba67d 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.whitefox.core.types.*; - import java.util.List; import java.util.Objects; @@ -28,18 +27,19 @@ public interface BaseOp { default Boolean isSupportedType(DataType valueType, EvaluatorVersion version) { if (version == EvaluatorVersion.V2) { return (valueType instanceof BooleanType - || valueType instanceof IntegerType - || valueType instanceof StringType - || valueType instanceof DateType - || valueType instanceof LongType - || valueType instanceof TimestampType - || valueType instanceof FloatType - || valueType instanceof DoubleType); - } else return (valueType instanceof BooleanType - || valueType instanceof IntegerType - || valueType instanceof StringType - || valueType instanceof DateType - || valueType instanceof LongType); + || valueType instanceof IntegerType + || valueType instanceof StringType + || valueType instanceof DateType + || valueType instanceof LongType + || valueType instanceof TimestampType + || valueType instanceof FloatType + || valueType instanceof DoubleType); + } else + return (valueType instanceof BooleanType + || valueType instanceof IntegerType + || valueType instanceof StringType + || valueType instanceof DateType + || valueType instanceof LongType); } Object eval(EvalContext ctx) throws PredicateException; @@ -65,14 +65,14 @@ default Boolean treeDepthExceeds(Integer depth) { } // marker interface for operator arity used for easier exception handling -interface Arity {}; +interface Arity {} +; // Represents a unary operation. interface UnaryOp extends Arity { // Validates number of children to be 1. default void validateChildren(List children) throws PredicateException { - if (children.size() != 1) - throw new PredicateValidationException(children.size(), this, 1); + if (children.size() != 1) throw new PredicateValidationException(children.size(), this, 1); try { children.get(0).validate(); } catch (PredicateException e) { @@ -84,8 +84,7 @@ default void validateChildren(List children) throws PredicateException { interface BinaryOp extends Arity { // Validates number of children to be 2. default void validateChildren(List children) throws PredicateException { - if (children.size() != 2) - throw new PredicateValidationException(children.size(), this, 2); + if (children.size() != 2) throw new PredicateValidationException(children.size(), this, 2); // otherwise cannot throw exception in method call of lambda for (BaseOp c : children) { diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java index d2c8002b7..446e0eef1 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java @@ -1,18 +1,17 @@ package io.whitefox.core.types.predicates; public enum BasePrimitiveTypeNames { - DATE("date"), - INT("int"), - FLOAT("float"), - DOUBLE("double"), - TIMESTAMP("timestamp"), - LONG("long"), - STRING("string") - ; + DATE("date"), + INT("int"), + FLOAT("float"), + DOUBLE("double"), + TIMESTAMP("timestamp"), + LONG("long"), + STRING("string"); - final public String value; + public final String value; - BasePrimitiveTypeNames(String value) { - this.value = value; - } + BasePrimitiveTypeNames(String value) { + this.value = value; + } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java b/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java index d71a936bd..c3af594d5 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java @@ -8,8 +8,6 @@ import io.whitefox.core.types.*; import java.io.IOException; - - public class DataTypeDeserializer extends StdDeserializer { // needed for jackson diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index 227f0a528..e7d2f3636 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -1,31 +1,25 @@ package io.whitefox.core.types.predicates; -import io.whitefox.core.ColumnRange; import io.whitefox.core.types.*; import java.sql.Date; import java.sql.Timestamp; import java.util.List; import java.util.Objects; -import java.util.Optional; - import org.apache.commons.lang3.tuple.Pair; // Only for partition values public class EvalHelper { - static LeafEvaluationResult validateAndGetRange(List children, EvalContext ctx) throws PredicateException { - var leftChild = (ColumnOp) children.get(0); - var columnRange = leftChild.evalExpectColumnRange(ctx); - - var rightChild = children.get(1); - var rightType = rightChild.evalExpectValueAndType(ctx).getRight(); - var rightVal = rightChild.evalExpectValueAndType(ctx).getLeft(); + static LeafEvaluationResult validateAndGetRange( + ColumnOp columnChild, LiteralOp literalChild, EvalContext ctx) throws PredicateException { + var columnRange = columnChild.evalExpectColumnRange(ctx); + var rightVal = literalChild.evalExpectValueAndType(ctx).getLeft(); return LeafEvaluationResult.createFromRange(Pair.of(columnRange, rightVal)); } - static LeafEvaluationResult validateAndGetTypeAndValue( - List children, EvalContext ctx) throws PredicateException { + static LeafEvaluationResult validateAndGetTypeAndValue(List children, EvalContext ctx) + throws PredicateException { var leftChild = children.get(0); var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); var leftVal = leftChild.evalExpectValueAndType(ctx).getLeft(); @@ -39,15 +33,22 @@ static LeafEvaluationResult validateAndGetTypeAndValue( throw new TypeMismatchException(leftType, rightType); } - if (leftVal == null && leftChild instanceof ColumnOp){ - return validateAndGetRange(children, ctx); + if (leftVal == null && leftChild instanceof ColumnOp) { + return validateAndGetRange((ColumnOp) leftChild, (LiteralOp) rightChild, ctx); + } + + // maybe better to enforce the Equal/LessThan... to explicitly require a column child and + // literal child + if (rightVal == null && rightChild instanceof ColumnOp) { + return validateAndGetRange((ColumnOp) rightChild, (LiteralOp) leftChild, ctx); } // We throw an exception for nulls, which will skip filtering. if (leftVal == null || rightVal == null) { throw new NullTypeException(leftChild, rightChild); } - return LeafEvaluationResult.createFromPartitionColumn(Pair.of(Pair.of(leftType, leftVal), Pair.of(rightType, rightVal))); + return LeafEvaluationResult.createFromPartitionColumn( + Pair.of(Pair.of(leftType, leftVal), Pair.of(rightType, rightVal))); } // Implements "equal" between two leaf operations. @@ -59,9 +60,8 @@ static Boolean equal(List children, EvalContext ctx) throws PredicateExc var value = range.getRight(); return columnRange.contains(value); }); - if (rangeEvaluation.isPresent()) - return rangeEvaluation.get(); - else if (leafEvaluationResult.partitionEvaluationResult.isPresent()){ + if (rangeEvaluation.isPresent()) return rangeEvaluation.get(); + else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { var typesAndValues = leafEvaluationResult.partitionEvaluationResult.get(); var leftType = typesAndValues.getLeft().getLeft(); var leftVal = typesAndValues.getLeft().getRight(); @@ -77,12 +77,8 @@ else if (leafEvaluationResult.partitionEvaluationResult.isPresent()){ return leftVal.equals(rightVal); } else if (DateType.DATE.equals(leftType)) { return Date.valueOf(leftVal).equals(Date.valueOf(rightVal)); - } - else - throw new TypeNotSupportedException(leftType); - } - else - throw new PredicateException(); + } else throw new TypeNotSupportedException(leftType); + } else throw new PredicateException(); } static Boolean lessThan(List children, EvalContext ctx) throws PredicateException { @@ -94,9 +90,8 @@ static Boolean lessThan(List children, EvalContext ctx) throws Predicate return columnRange.canBeLess(value); }); - if (rangeEvaluation.isPresent()) - return rangeEvaluation.get(); - else if (leafEvaluationResult.partitionEvaluationResult.isPresent()){ + if (rangeEvaluation.isPresent()) return rangeEvaluation.get(); + else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { var typesAndValues = leafEvaluationResult.partitionEvaluationResult.get(); var leftType = typesAndValues.getLeft().getLeft(); var leftVal = typesAndValues.getLeft().getRight(); @@ -110,12 +105,8 @@ else if (leafEvaluationResult.partitionEvaluationResult.isPresent()){ return leftVal.compareTo(rightVal) < 0; } else if (DateType.DATE.equals(leftType)) { return Date.valueOf(leftVal).before(Date.valueOf(rightVal)); - } - else - throw new TypeNotSupportedException(leftType); - } - else - throw new PredicateException(); + } else throw new TypeNotSupportedException(leftType); + } else throw new PredicateException(); } // Validates that the specified value is in the correct format. diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvaluatorVersion.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvaluatorVersion.java index 603e6fb5d..14828873f 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvaluatorVersion.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvaluatorVersion.java @@ -1,12 +1,12 @@ package io.whitefox.core.types.predicates; public enum EvaluatorVersion { - V1("v1"), - V2("v2"); + V1("v1"), + V2("v2"); - public final String value; + public final String value; - EvaluatorVersion(String value) { - this.value = value; - } + EvaluatorVersion(String value) { + this.value = value; + } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafEvaluationResult.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafEvaluationResult.java index 047db20b4..38736a0f9 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafEvaluationResult.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafEvaluationResult.java @@ -2,27 +2,28 @@ import io.whitefox.core.ColumnRange; import io.whitefox.core.types.DataType; -import org.apache.commons.lang3.tuple.Pair; - import java.util.Optional; +import org.apache.commons.lang3.tuple.Pair; public class LeafEvaluationResult { - Optional> rangeEvaluationResult; - Optional, Pair>> partitionEvaluationResult; - - public LeafEvaluationResult(Optional> rangeEvaluationResult, Optional, Pair>> partitionEvaluationResult) { - this.rangeEvaluationResult = rangeEvaluationResult; - this.partitionEvaluationResult = partitionEvaluationResult; - } - - public static LeafEvaluationResult createFromRange(Pair rangeEvaluationResult) { - return new LeafEvaluationResult(Optional.of(rangeEvaluationResult), Optional.empty()); - } - - public static LeafEvaluationResult createFromPartitionColumn(Pair, Pair> partitionEvaluationResult) { - return new LeafEvaluationResult(Optional.empty(), Optional.of(partitionEvaluationResult)); - } - - + Optional> rangeEvaluationResult; + Optional, Pair>> partitionEvaluationResult; + + public LeafEvaluationResult( + Optional> rangeEvaluationResult, + Optional, Pair>> partitionEvaluationResult) { + this.rangeEvaluationResult = rangeEvaluationResult; + this.partitionEvaluationResult = partitionEvaluationResult; + } + + public static LeafEvaluationResult createFromRange( + Pair rangeEvaluationResult) { + return new LeafEvaluationResult(Optional.of(rangeEvaluationResult), Optional.empty()); + } + + public static LeafEvaluationResult createFromPartitionColumn( + Pair, Pair> partitionEvaluationResult) { + return new LeafEvaluationResult(Optional.empty(), Optional.of(partitionEvaluationResult)); + } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java index 71b36442e..b570ddc1d 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -1,18 +1,17 @@ package io.whitefox.core.types.predicates; +import static io.whitefox.core.JsonPredicatesUtils.createColumnRange; +import static io.whitefox.core.types.predicates.EvaluatorVersion.V1; + import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.whitefox.core.ColumnRange; import io.whitefox.core.types.BooleanType; import io.whitefox.core.types.DataType; -import org.apache.commons.lang3.tuple.Pair; - import java.util.List; import java.util.Objects; - -import static io.whitefox.core.JsonPredicatesUtils.createColumnRange; -import static io.whitefox.core.types.predicates.EvaluatorVersion.V1; +import org.apache.commons.lang3.tuple.Pair; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "op") @JsonSubTypes({ @@ -72,7 +71,7 @@ public Boolean evalExpectBoolean(EvalContext ctx) { return Boolean.valueOf(resolve(ctx)); } - public ColumnRange evalExpectColumnRange(EvalContext ctx) { + public ColumnRange evalExpectColumnRange(EvalContext ctx) throws NonExistingColumnException { return createColumnRange(name, ctx, valueType); } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NonExistingColumnException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NonExistingColumnException.java new file mode 100644 index 000000000..725a53e92 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NonExistingColumnException.java @@ -0,0 +1,14 @@ +package io.whitefox.core.types.predicates; + +public class NonExistingColumnException extends PredicateException { + private final String name; + + public NonExistingColumnException(String name) { + this.name = name; + } + + @Override + public String getMessage() { + return "Column " + name + " does not exist in the file statistics"; + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java index 3473aaf2c..90e062a7f 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; - import java.util.List; import java.util.stream.Collectors; diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NullTypeException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NullTypeException.java index 8d2fb26b6..5f53116bf 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/NullTypeException.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NullTypeException.java @@ -2,19 +2,19 @@ public class NullTypeException extends PredicateException { - final private BaseOp leftChild; - final private BaseOp rightChild; + private final BaseOp leftChild; + private final BaseOp rightChild; + public NullTypeException(BaseOp leftChild, BaseOp rightChild) { + this.leftChild = leftChild; + this.rightChild = rightChild; + } - public NullTypeException(BaseOp leftChild, BaseOp rightChild) { - this.leftChild = leftChild; - this.rightChild = rightChild; - } - - @Override - public String getMessage() { - // TODO: Currently means that the column is not in the partition columns - // Expected is to mean that it is not present at all(use file stats) - return "Comparison with a null value is not supported: " + leftChild.getClass() + " and " + rightChild.getClass(); - } + @Override + public String getMessage() { + // TODO: Currently means that the column is not in the partition columns + // Expected is to mean that it is not present at all(use file stats) + return "Comparison with a null value is not supported: " + leftChild.getClass() + " and " + + rightChild.getClass(); + } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateException.java index 585498dba..701105531 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateException.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateException.java @@ -1,7 +1,3 @@ package io.whitefox.core.types.predicates; - -public class PredicateException extends Exception { - -} - +public class PredicateException extends Exception {} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java index 3ca1283f4..8ece1d34b 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java @@ -1,20 +1,19 @@ package io.whitefox.core.types.predicates; import com.fasterxml.jackson.core.JsonProcessingException; - import java.util.Arrays; public class PredicateParsingException extends PredicateException { - private final JsonProcessingException cause; + private final JsonProcessingException cause; - public PredicateParsingException(JsonProcessingException cause) { - this.cause = cause; - } + public PredicateParsingException(JsonProcessingException cause) { + this.cause = cause; + } - @Override - public String getMessage() { - return "Parsing of predicate failed due to: " + cause.getMessage() + "\n Stack trace: " + Arrays.toString(cause.getStackTrace()); - } + @Override + public String getMessage() { + return "Parsing of predicate failed due to: " + cause.getMessage() + "\n Stack trace: " + + Arrays.toString(cause.getStackTrace()); + } } - diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateValidationException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateValidationException.java index 050a62042..b77ac6153 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateValidationException.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateValidationException.java @@ -2,23 +2,24 @@ public class PredicateValidationException extends PredicateException { + private final int actualNumOfChildren; + private final Arity op; + private final int expectedNumOfChildren; - final private int actualNumOfChildren; - final private Arity op; - final private int expectedNumOfChildren; + public PredicateValidationException( + int actualNumOfChildren, Arity op, int expectedNumOfChildren) { + this.actualNumOfChildren = actualNumOfChildren; + this.op = op; + this.expectedNumOfChildren = expectedNumOfChildren; + } - public PredicateValidationException(int actualNumOfChildren, Arity op, int expectedNumOfChildren) { - this.actualNumOfChildren = actualNumOfChildren; - this.op = op; - this.expectedNumOfChildren = expectedNumOfChildren; - } - - - @Override - public String getMessage() { - if (op instanceof NaryOp) - return op + " : expected at least " + expectedNumOfChildren + " children, but found " + actualNumOfChildren + " children"; - else - return op + " : expected " + expectedNumOfChildren + " children, but found " + actualNumOfChildren + " children"; - } + @Override + public String getMessage() { + if (op instanceof NaryOp) + return op + " : expected at least " + expectedNumOfChildren + " children, but found " + + actualNumOfChildren + " children"; + else + return op + " : expected " + expectedNumOfChildren + " children, but found " + + actualNumOfChildren + " children"; + } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/TypeMismatchException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/TypeMismatchException.java index aab90263a..58093d5f5 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/TypeMismatchException.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/TypeMismatchException.java @@ -4,17 +4,16 @@ public class TypeMismatchException extends PredicateException { - final private DataType lType; - final private DataType rType; + private final DataType lType; + private final DataType rType; - public TypeMismatchException(DataType lType, DataType rType) { - this.lType = lType; - this.rType = rType; - } - - @Override - public String getMessage() { - return "Type are not matching between: " + lType.toString() + "and " + rType; - } + public TypeMismatchException(DataType lType, DataType rType) { + this.lType = lType; + this.rType = rType; + } + @Override + public String getMessage() { + return "Type are not matching between: " + lType.toString() + "and " + rType; + } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/TypeNotSupportedException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/TypeNotSupportedException.java index 59f08e325..6415e93dd 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/TypeNotSupportedException.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/TypeNotSupportedException.java @@ -3,14 +3,14 @@ import io.whitefox.core.types.DataType; public class TypeNotSupportedException extends PredicateException { - final private DataType type; + private final DataType type; - public TypeNotSupportedException(DataType type) { - this.type = type; - } + public TypeNotSupportedException(DataType type) { + this.type = type; + } - @Override - public String getMessage() { - return "Unsupported type: " + type.toString(); - } + @Override + public String getMessage() { + return "Unsupported type: " + type.toString(); + } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/WrongExpectedTypeException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/WrongExpectedTypeException.java index 2225a1002..391f655d4 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/WrongExpectedTypeException.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/WrongExpectedTypeException.java @@ -2,16 +2,17 @@ public class WrongExpectedTypeException extends PredicateException { - final private Object evaluationResult; - final private Class expectedType; + private final Object evaluationResult; + private final Class expectedType; - public WrongExpectedTypeException(Object evaluationResult, Class expectedType) { - this.evaluationResult = evaluationResult; - this.expectedType = expectedType; - } + public WrongExpectedTypeException(Object evaluationResult, Class expectedType) { + this.evaluationResult = evaluationResult; + this.expectedType = expectedType; + } - @Override - public String getMessage() { - return "Evaluation of a Root or Non-Leaf predicate is expected to be of " + expectedType + " type, instead got: " + evaluationResult.getClass(); - } + @Override + public String getMessage() { + return "Evaluation of a Root or Non-Leaf predicate is expected to be of " + expectedType + + " type, instead got: " + evaluationResult.getClass(); + } } diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index 4f0766901..1dd8047b2 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -12,7 +12,6 @@ import java.sql.Timestamp; import java.time.format.DateTimeParseException; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.concurrent.ExecutionException; import org.junit.jupiter.api.Test; @@ -83,8 +82,7 @@ void queryTableWithoutPredicate() { var PTable = new SharedTable( "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); - var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(), Optional.empty()); + var request = new ReadTableRequest.ReadTableCurrentVersion(List.of(), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.protocol(), new Protocol(Optional.of(1))); assertEquals(response.other().size(), 9); @@ -93,18 +91,18 @@ void queryTableWithoutPredicate() { @Test void queryTableWithJsonPredicate() { var predicate = "{" - + " \"op\":\"equal\",\n" - + " \"children\":[\n" - + " {\"op\":\"column\",\"name\":\"date\",\"valueType\":\"date\"},\n" - + " {\"op\":\"literal\",\"value\":\"2021-08-15\",\"valueType\":\"date\"}\n" - + " ]\n" - + "}"; + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"date\",\"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-08-15\",\"valueType\":\"date\"}\n" + + " ]\n" + + "}"; var PTable = new SharedTable( - "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); + "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); - var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(predicate), Optional.empty()); + var request = + new ReadTableRequest.ReadTableCurrentVersion(List.of(predicate), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.other().size(), 4); } @@ -112,18 +110,18 @@ void queryTableWithJsonPredicate() { @Test void queryTableWithInvalidJsonPredicate() { var predicate = "{" - + " \"op\":\"equal\",\n" - + " \"children\":[\n" - + " {\"op\":\"column\",\"name\":\"dating\",\"valueType\":\"date\"},\n" - + " {\"op\":\"literal\",\"value\":\"2021-08-15\",\"valueType\":\"date\"}\n" - + " ]\n" - + "}"; + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"dating\",\"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-08-15\",\"valueType\":\"date\"}\n" + + " ]\n" + + "}"; var PTable = new SharedTable( - "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); + "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); - var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(predicate), Optional.empty()); + var request = + new ReadTableRequest.ReadTableCurrentVersion(List.of(predicate), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.other().size(), 9); } @@ -132,20 +130,18 @@ void queryTableWithInvalidJsonPredicate() { void queryTableWithColumnRangePredicate() { var tableName = "partitioned-delta-table-with-multiple-columns"; var predicate = "{" - + " \"op\":\"lessThan\",\n" - + " \"children\":[\n" - + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" - + " {\"op\":\"literal\",\"value\":\"4\",\"valueType\":\"int\"}\n" - + " ]\n" - + "}"; - - var PTable = new SharedTable( - tableName, "default", "share1", deltaTable(tableName)); + + " \"op\":\"lessThan\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + + " {\"op\":\"literal\",\"value\":\"4\",\"valueType\":\"int\"}\n" + + " ]\n" + + "}"; + + var PTable = new SharedTable(tableName, "default", "share1", deltaTable(tableName)); var DTable = DeltaSharedTable.of(PTable); - var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(predicate), Optional.empty()); + var request = + new ReadTableRequest.ReadTableCurrentVersion(List.of(predicate), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.other().size(), 1); } - } diff --git a/server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java b/server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java index f0a874186..14d6180a2 100644 --- a/server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java @@ -3,18 +3,12 @@ import static io.whitefox.DeltaTestUtils.deltaTable; import static io.whitefox.DeltaTestUtils.deltaTableUri; -import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.standalone.DeltaLog; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - import io.delta.standalone.actions.AddFile; import io.whitefox.core.JsonPredicatesUtils; import io.whitefox.core.SharedTable; import io.whitefox.core.types.predicates.*; -import jakarta.inject.Inject; +import java.util.ArrayList; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Test; @@ -28,7 +22,6 @@ void testCreateEvalContext() throws PredicateParsingException { "share1", deltaTable("partitioned-delta-table-with-multiple-columns")); - var log = DeltaLog.forTable( new Configuration(), deltaTableUri("partitioned-delta-table-with-multiple-columns")); var contexts = new ArrayList(); diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java index c6dc4b668..975dc81af 100644 --- a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java @@ -1,11 +1,10 @@ package io.whitefox.core.types.predicates; -import com.fasterxml.jackson.core.JsonProcessingException; +import static org.junit.jupiter.api.Assertions.assertThrows; + import io.whitefox.core.JsonPredicatesUtils; import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertThrows; - public class PredicateParsingTest { @Test @@ -52,22 +51,23 @@ void testParsingOfNested() throws PredicateException { @Test void testCustomExceptionOnBadJson() { var predicate = "{\n" + " \"op\":\"and\",\n" - + " \"children\":[\n" - + " {\n" - + " \"op\":\"equals\",\n" - + " \"children\":[\n" - + " {\"op\":\"columna\",\"name\":\"hireDate\",\"valueType\":\"date\"},\n" - + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" - + " ]\n" - + " },\n" - + " {\n" - + " \"op\":\"lessThans\",\"children\":[\n" - + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" - + " {\"op\":\"literal\",\"value\":\"25\",\"valueType\":\"int\"}\n" - + " ]\n" - + " }\n" - + " ]\n" - + "}"; - assertThrows(PredicateParsingException.class , () -> JsonPredicatesUtils.parsePredicate(predicate)); + + " \"children\":[\n" + + " {\n" + + " \"op\":\"equals\",\n" + + " \"children\":[\n" + + " {\"op\":\"columna\",\"name\":\"hireDate\",\"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"op\":\"lessThans\",\"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + + " {\"op\":\"literal\",\"value\":\"25\",\"valueType\":\"int\"}\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}"; + assertThrows( + PredicateParsingException.class, () -> JsonPredicatesUtils.parsePredicate(predicate)); } } From 2b4597c4b01ea7aa72ba7c73a75ee5fbea16e296 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Tue, 19 Dec 2023 02:29:18 +0100 Subject: [PATCH 11/30] sql filters parsing --- server/core/build.gradle.kts | 3 + .../io/whitefox/core/JsonPredicatesUtils.java | 45 +++++++- .../core/services/DeltaSharedTable.java | 10 +- .../core/types/predicates/ColumnOp.java | 72 ++++++++++++ .../core/types/predicates/EvalHelper.java | 4 +- .../core/types/predicates/LeafOp.java | 108 +----------------- .../core/types/predicates/LiteralOp.java | 50 ++++++++ .../core/types/predicates/NonLeafOp.java | 35 ++++++ .../predicates/PredicateParsingTest.java | 6 +- 9 files changed, 212 insertions(+), 121 deletions(-) create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/LiteralOp.java diff --git a/server/core/build.gradle.kts b/server/core/build.gradle.kts index 5eb68dbe3..926cb1511 100644 --- a/server/core/build.gradle.kts +++ b/server/core/build.gradle.kts @@ -43,6 +43,9 @@ dependencies { compileOnly(String.format("com.amazonaws:aws-java-sdk-s3:%s", awsSdkVersion)) implementation(String.format("org.apache.hadoop:hadoop-aws:%s", hadoopVersion)) + //PREDICATE PARSER + implementation("com.github.jsqlparser:jsqlparser:4.7") + // TEST testImplementation("org.junit.jupiter:junit-jupiter") testImplementation("io.quarkus:quarkus-arc") diff --git a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java index 67d8ac4db..4afdfd944 100644 --- a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java +++ b/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java @@ -3,19 +3,27 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.standalone.actions.AddFile; +import io.delta.standalone.expressions.IsNull; import io.whitefox.core.types.DataType; -import io.whitefox.core.types.predicates.BaseOp; -import io.whitefox.core.types.predicates.EvalContext; -import io.whitefox.core.types.predicates.NonExistingColumnException; -import io.whitefox.core.types.predicates.PredicateParsingException; +import io.whitefox.core.types.DateType; +import io.whitefox.core.types.predicates.*; + +import java.util.List; +import java.util.Objects; import java.util.Optional; + +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.operators.relational.IsNullExpression; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.select.PlainSelect; import org.apache.commons.lang3.tuple.Pair; public class JsonPredicatesUtils { private static final ObjectMapper objectMapper = DeltaObjectMapper.getInstance(); - public static BaseOp parsePredicate(String predicate) throws PredicateParsingException { + public static BaseOp parseJsonPredicate(String predicate) throws PredicateParsingException { try { return objectMapper.readValue(predicate, BaseOp.class); } catch (JsonProcessingException e) { @@ -23,6 +31,33 @@ public static BaseOp parsePredicate(String predicate) throws PredicateParsingExc } } + public static BaseOp parseSqlPredicate(String predicate, DataType dataType) throws JSQLParserException, PredicateException { + var expression = CCJSqlParserUtil.parseCondExpression(predicate); + if (expression instanceof IsNullExpression){ + var isNullExpression = (IsNullExpression) expression; + String column = isNullExpression.getLeftExpression().getASTNode().jjtGetFirstToken().toString(); + var colOp = new ColumnOp(column, dataType); + var children = List.of((LeafOp) colOp); + var operator = "isnull"; + return NonLeafOp.createPartitionFilter(children, operator); + } + else if (expression instanceof BinaryExpression) { + BinaryExpression binaryExpression = (BinaryExpression) expression; + String column = binaryExpression.getLeftExpression().toString(); + String operator = binaryExpression.getStringExpression(); + String value = binaryExpression.getRightExpression().toString(); + var colOp = new ColumnOp(column, dataType); + var litOp = new LiteralOp(value, dataType); + var children = List.of(colOp, litOp); + return NonLeafOp.createPartitionFilter(children, operator); + } + // TODO: PARSING FAIL on sql; + else + throw new PredicateException(); + } + + + public static ColumnRange createColumnRange(String name, EvalContext ctx, DataType valueType) throws NonExistingColumnException { var fileStats = ctx.getStatsValues(); diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index 81198a125..0ce1240d4 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -88,9 +88,9 @@ public Optional getTableVersion(Optional startingTimestamp) { return getSnapshot(startingTimestamp).map(Snapshot::getVersion); } - private boolean evaluatePredicate(String predicate, EvalContext ctx, AddFile f) { + private boolean evaluateJsonPredicate(String predicate, EvalContext ctx, AddFile f) { try { - var parsedPredicate = JsonPredicatesUtils.parsePredicate(predicate); + var parsedPredicate = JsonPredicatesUtils.parseJsonPredicate(predicate); return parsedPredicate.evalExpectBoolean(ctx); } catch (PredicateException e) { logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage()); @@ -101,14 +101,14 @@ private boolean evaluatePredicate(String predicate, EvalContext ctx, AddFile f) } } - public boolean filterFilesBasedOnPredicates(List predicates, AddFile f) { + public boolean filterFilesBasedOnJsonPredicates(List predicates, AddFile f) { // if there are no predicates return all possible files if (predicates == null) { return true; } try { var ctx = JsonPredicatesUtils.createEvalContext(f); - return predicates.stream().allMatch(p -> evaluatePredicate(p, ctx, f)); + return predicates.stream().allMatch(p -> evaluateJsonPredicate(p, ctx, f)); } catch (PredicateException e) { logger.debug("Caught exception: " + e.getMessage()); logger.info("File: " + f.getPath() @@ -139,7 +139,7 @@ public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { new Protocol(Optional.of(1)), metadataFromSnapshot(snapshot), snapshot.getAllFiles().stream() - .filter(f -> filterFilesBasedOnPredicates(predicates, f)) + .filter(f -> filterFilesBasedOnJsonPredicates(predicates, f)) .map(f -> new TableFileToBeSigned( location() + "/" + f.getPath(), f.getSize(), diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java new file mode 100644 index 000000000..4f5ff41a5 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java @@ -0,0 +1,72 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.whitefox.core.ColumnRange; +import io.whitefox.core.types.BooleanType; +import io.whitefox.core.types.DataType; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.Objects; + +import static io.whitefox.core.JsonPredicatesUtils.createColumnRange; +import static io.whitefox.core.types.predicates.EvaluatorVersion.V1; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "column") +public class ColumnOp extends LeafOp { + + @JsonProperty("name") + String name; + + public ColumnOp() { + super(); + } + + public ColumnOp(String name, DataType valueType) { + this.name = name; + this.valueType = valueType; + } + + // Determine if the column value is null. + @Override + public Boolean isNull(EvalContext ctx) { + return resolve(ctx) == null; + } + + @Override + public Boolean evalExpectBoolean(EvalContext ctx) { + if (!Objects.equals(valueType, BooleanType.BOOLEAN)) { + throw new IllegalArgumentException("Unsupported type for boolean evaluation: " + valueType); + } + return Boolean.valueOf(resolve(ctx)); + } + + public ColumnRange evalExpectColumnRange(EvalContext ctx) throws NonExistingColumnException { + return createColumnRange(name, ctx, valueType); + } + + @Override + public DataType getOpValueType() { + return valueType; + } + + @Override + public Object eval(EvalContext ctx) { + // TODO: handle case of null column + column ranges + return Pair.of(resolve(ctx), valueType); + } + + public void validate() throws PredicateException { + if (name == null) { + throw new IllegalArgumentException("Name must be specified: " + this); + } + if (!this.isSupportedType(valueType, V1)) { + throw new TypeNotSupportedException(valueType); + } + } + + private String resolve(EvalContext ctx) { + // TODO: handle case of null column + column ranges + return ctx.partitionValues.getOrDefault(name, null); + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index e7d2f3636..1f7db4536 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -10,7 +10,7 @@ // Only for partition values public class EvalHelper { - static LeafEvaluationResult validateAndGetRange( + private static LeafEvaluationResult validateAndGetRange( ColumnOp columnChild, LiteralOp literalChild, EvalContext ctx) throws PredicateException { var columnRange = columnChild.evalExpectColumnRange(ctx); var rightVal = literalChild.evalExpectValueAndType(ctx).getLeft(); @@ -18,7 +18,7 @@ static LeafEvaluationResult validateAndGetRange( return LeafEvaluationResult.createFromRange(Pair.of(columnRange, rightVal)); } - static LeafEvaluationResult validateAndGetTypeAndValue(List children, EvalContext ctx) + private static LeafEvaluationResult validateAndGetTypeAndValue(List children, EvalContext ctx) throws PredicateException { var leftChild = children.get(0); var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java index b570ddc1d..a66a937f9 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -1,18 +1,13 @@ package io.whitefox.core.types.predicates; -import static io.whitefox.core.JsonPredicatesUtils.createColumnRange; -import static io.whitefox.core.types.predicates.EvaluatorVersion.V1; - import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.whitefox.core.ColumnRange; -import io.whitefox.core.types.BooleanType; import io.whitefox.core.types.DataType; -import java.util.List; -import java.util.Objects; import org.apache.commons.lang3.tuple.Pair; +import java.util.List; + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "op") @JsonSubTypes({ @JsonSubTypes.Type(value = ColumnOp.class, name = "column"), @@ -42,102 +37,3 @@ public List getAllChildren() { abstract DataType getOpValueType(); } -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "column") -class ColumnOp extends LeafOp { - - @JsonProperty("name") - String name; - - public ColumnOp() { - super(); - } - - public ColumnOp(String name, DataType valueType) { - this.name = name; - this.valueType = valueType; - } - - // Determine if the column value is null. - @Override - public Boolean isNull(EvalContext ctx) { - return resolve(ctx) == null; - } - - @Override - public Boolean evalExpectBoolean(EvalContext ctx) { - if (!Objects.equals(valueType, BooleanType.BOOLEAN)) { - throw new IllegalArgumentException("Unsupported type for boolean evaluation: " + valueType); - } - return Boolean.valueOf(resolve(ctx)); - } - - public ColumnRange evalExpectColumnRange(EvalContext ctx) throws NonExistingColumnException { - return createColumnRange(name, ctx, valueType); - } - - @Override - public DataType getOpValueType() { - return valueType; - } - - @Override - public Object eval(EvalContext ctx) { - // TODO: handle case of null column + column ranges - return Pair.of(resolve(ctx), valueType); - } - - public void validate() throws PredicateException { - if (name == null) { - throw new IllegalArgumentException("Name must be specified: " + this); - } - if (!this.isSupportedType(valueType, V1)) { - throw new TypeNotSupportedException(valueType); - } - } - - private String resolve(EvalContext ctx) { - // TODO: handle case of null column + column ranges - return ctx.partitionValues.getOrDefault(name, null); - } -} - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "literal") -class LiteralOp extends LeafOp { - @JsonProperty("value") - String value; - - @Override - public void validate() throws PredicateException { - if (value == null) { - throw new IllegalArgumentException("Value must be specified: " + this); - } - if (!isSupportedType(valueType, V1)) { - throw new IllegalArgumentException("Unsupported type: " + valueType); - } - EvalHelper.validateValue(value, valueType); - } - - @Override - public Object eval(EvalContext ctx) { - return Pair.of(value, valueType); - } - - public LiteralOp() { - super(); - } - - public LiteralOp(String value, DataType valueType) { - this.value = value; - this.valueType = valueType; - } - - @Override - public Boolean isNull(EvalContext ctx) { - return false; - } - - @Override - public DataType getOpValueType() { - return valueType; - } -} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LiteralOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LiteralOp.java new file mode 100644 index 000000000..fe4d353ce --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LiteralOp.java @@ -0,0 +1,50 @@ +package io.whitefox.core.types.predicates; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.whitefox.core.types.DataType; +import org.apache.commons.lang3.tuple.Pair; + +import static io.whitefox.core.types.predicates.EvaluatorVersion.V1; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "literal") +public +class LiteralOp extends LeafOp { + @JsonProperty("value") + String value; + + @Override + public void validate() throws PredicateException { + if (value == null) { + throw new IllegalArgumentException("Value must be specified: " + this); + } + if (!isSupportedType(valueType, V1)) { + throw new IllegalArgumentException("Unsupported type: " + valueType); + } + EvalHelper.validateValue(value, valueType); + } + + @Override + public Object eval(EvalContext ctx) { + return Pair.of(value, valueType); + } + + public LiteralOp() { + super(); + } + + public LiteralOp(String value, DataType valueType) { + this.value = value; + this.valueType = valueType; + } + + @Override + public Boolean isNull(EvalContext ctx) { + return false; + } + + @Override + public DataType getOpValueType() { + return valueType; + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java index 90e062a7f..e40655a9a 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java @@ -3,7 +3,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.whitefox.core.types.DateType; +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.statement.select.PlainSelect; + import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; // Represents a non-leaf operation. @@ -24,6 +31,26 @@ public abstract class NonLeafOp implements BaseOp { @JsonProperty("children") List children; + public static NonLeafOp createPartitionFilter(List children, String operator) throws PredicateException { + switch (operator) { + case "=": + return new EqualOp(children); + case "<": + return new LessThanOp(children); + case "<=": + return new LessThanOrEqualOp(children); + case ">": + return new GreaterThanOp(children); + case ">=": + return new GreaterThanOrEqualOp(children); + case "isnull": + return new IsNullOp(children); + default: + // TODO: add not supported sql exception + throw new PredicateException(); + } + } + public List getAllChildren() { // TODO flat map every child return List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList()); @@ -156,6 +183,10 @@ public GreaterThanOp() { super(); } + public GreaterThanOp(List children) { + this.children = children; + } + @Override public void validate() throws PredicateException { validateChildren( @@ -183,6 +214,10 @@ public GreaterThanOrEqualOp() { super(); } + public GreaterThanOrEqualOp(List children) { + this.children = children; + } + @Override public void validate() throws PredicateException { validateChildren( diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java index 975dc81af..647e6184e 100644 --- a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java @@ -16,7 +16,7 @@ void testParsingOfEqual() throws PredicateException { + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + " ]\n" + "}"; - var op = JsonPredicatesUtils.parsePredicate(predicate); + var op = JsonPredicatesUtils.parseJsonPredicate(predicate); op.validate(); assert (op instanceof EqualOp); assert (((EqualOp) op).children.size() == 2); @@ -41,7 +41,7 @@ void testParsingOfNested() throws PredicateException { + " }\n" + " ]\n" + "}"; - var op = JsonPredicatesUtils.parsePredicate(predicate); + var op = JsonPredicatesUtils.parseJsonPredicate(predicate); op.validate(); assert (op instanceof AndOp); assert (((AndOp) op).children.size() == 2); @@ -68,6 +68,6 @@ void testCustomExceptionOnBadJson() { + " ]\n" + "}"; assertThrows( - PredicateParsingException.class, () -> JsonPredicatesUtils.parsePredicate(predicate)); + PredicateParsingException.class, () -> JsonPredicatesUtils.parseJsonPredicate(predicate)); } } From 453445893051e18aaad390ee81f5d2065f84c3de Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 22 Dec 2023 13:49:55 +0100 Subject: [PATCH 12/30] wip- sql predicates without error handling --- protocol/delta-sharing-protocol-api.yml | 4 +- .../api/deltasharing/DeltaMappers.java | 6 +- .../java/io/whitefox/core/ColumnRange.java | 14 +-- ...edicatesUtils.java => PredicateUtils.java} | 69 ++++++++------ .../io/whitefox/core/ReadTableRequest.java | 59 +++++++++--- .../core/services/DeltaSharedTable.java | 53 +++++++++-- .../core/types/predicates/ColumnOp.java | 95 +++++++++---------- .../core/types/predicates/EvalHelper.java | 6 +- .../ExpressionNotSupportedException.java | 14 +++ .../core/types/predicates/LeafOp.java | 4 +- .../core/types/predicates/LiteralOp.java | 77 ++++++++------- .../core/types/predicates/NonLeafOp.java | 10 +- .../predicates/PredicateParsingException.java | 5 +- .../core/services/DeltaSharedTableTest.java | 41 ++++++-- ...UtilsTest.java => PredicateUtilsTest.java} | 6 +- .../predicates/PredicateParsingTest.java | 8 +- 16 files changed, 290 insertions(+), 181 deletions(-) rename server/core/src/main/java/io/whitefox/core/{JsonPredicatesUtils.java => PredicateUtils.java} (50%) create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/ExpressionNotSupportedException.java rename server/core/src/test/java/io/whitefox/core/types/{JsonPredicatesUtilsTest.java => PredicateUtilsTest.java} (87%) diff --git a/protocol/delta-sharing-protocol-api.yml b/protocol/delta-sharing-protocol-api.yml index 8cd1e94e8..1da49591b 100644 --- a/protocol/delta-sharing-protocol-api.yml +++ b/protocol/delta-sharing-protocol-api.yml @@ -670,7 +670,7 @@ components: items: type: string jsonPredicateHints: - type: string + type: array description: | query predicates on partition columns specified using a structured JSON format. When it’s present, the server will try to use the predicates to filter table's @@ -680,6 +680,8 @@ components: If the server encounters any errors during predicate processing (for example, invalid syntax or non existing columns), it will skip filtering and return all the files. When it’s absent, the server will return all the files in the table. + items: + type: string # properties: # op: # $ref: '#/components/schemas/Ops' diff --git a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java index 6948065c7..f85f5bba3 100644 --- a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java +++ b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java @@ -34,16 +34,20 @@ public static ReadTableRequest api2ReadTableRequest(QueryRequest request) { } else if (request.getVersion() != null && request.getTimestamp() == null) { return new ReadTableRequest.ReadTableVersion( request.getPredicateHints(), + request.getJsonPredicateHints(), Optional.ofNullable(request.getLimitHint()), request.getVersion()); } else if (request.getVersion() == null && request.getTimestamp() != null) { return new ReadTableRequest.ReadTableAsOfTimestamp( request.getPredicateHints(), + request.getJsonPredicateHints(), Optional.ofNullable(request.getLimitHint()), CommonMappers.parseTimestamp(request.getTimestamp())); } else if (request.getVersion() == null && request.getTimestamp() == null) { return new ReadTableRequest.ReadTableCurrentVersion( - request.getPredicateHints(), Optional.ofNullable(request.getLimitHint())); + request.getPredicateHints(), + request.getJsonPredicateHints(), + Optional.ofNullable(request.getLimitHint())); } else { throw new IllegalArgumentException("Cannot specify both version and timestamp"); } diff --git a/server/core/src/main/java/io/whitefox/core/ColumnRange.java b/server/core/src/main/java/io/whitefox/core/ColumnRange.java index 3efa1d1c3..4ea3c2361 100644 --- a/server/core/src/main/java/io/whitefox/core/ColumnRange.java +++ b/server/core/src/main/java/io/whitefox/core/ColumnRange.java @@ -34,7 +34,7 @@ private Boolean typedContains(String point) { return (c1 <= 0 && c2 >= 0); } else if (valueType instanceof TimestampType) { var c1 = Timestamp.valueOf(minVal).before(Timestamp.valueOf(point)); - var c2 = Timestamp.valueOf(maxVal).before(Timestamp.valueOf(point)); + var c2 = Timestamp.valueOf(maxVal).after(Timestamp.valueOf(point)); return c1 && c2; } else if (valueType instanceof FloatType) { var c1 = Float.compare(Float.parseFloat(minVal), Float.parseFloat(point)); @@ -46,7 +46,7 @@ private Boolean typedContains(String point) { return (c1 <= 0 && c2 >= 0); } else if (valueType instanceof DateType) { var c1 = Date.valueOf(minVal).before(Date.valueOf(point)); - var c2 = Date.valueOf(maxVal).before(Date.valueOf(point)); + var c2 = Date.valueOf(maxVal).after(Date.valueOf(point)); return c1 && c2; } else if (valueType instanceof BooleanType) { var c1 = Boolean.parseBoolean(minVal) == Boolean.parseBoolean(point); @@ -59,15 +59,6 @@ private Boolean typedContains(String point) { } } - public static void main(String[] args) { - var minVal = "4"; - var point = "5"; - var maxVal = "8"; - - var cr = new ColumnRange(minVal, maxVal, IntegerType.INTEGER); - cr.typedLessThan(point); - } - private Boolean typedLessThan(String point) { if (valueType instanceof IntegerType) { var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point)); @@ -85,7 +76,6 @@ private Boolean typedLessThan(String point) { return (c1 < 0); } else if (valueType instanceof DateType) { return Date.valueOf(minVal).before(Date.valueOf(point)); - } else { var c = minVal.compareTo(point); return (c < 0); diff --git a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java b/server/core/src/main/java/io/whitefox/core/PredicateUtils.java similarity index 50% rename from server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java rename to server/core/src/main/java/io/whitefox/core/PredicateUtils.java index 4afdfd944..2e0f6dfb4 100644 --- a/server/core/src/main/java/io/whitefox/core/JsonPredicatesUtils.java +++ b/server/core/src/main/java/io/whitefox/core/PredicateUtils.java @@ -3,23 +3,19 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import io.delta.standalone.actions.AddFile; -import io.delta.standalone.expressions.IsNull; import io.whitefox.core.types.DataType; -import io.whitefox.core.types.DateType; import io.whitefox.core.types.predicates.*; - import java.util.List; -import java.util.Objects; import java.util.Optional; - import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.BinaryExpression; +import net.sf.jsqlparser.expression.Expression; +import net.sf.jsqlparser.expression.StringValue; import net.sf.jsqlparser.expression.operators.relational.IsNullExpression; import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.statement.select.PlainSelect; import org.apache.commons.lang3.tuple.Pair; -public class JsonPredicatesUtils { +public class PredicateUtils { private static final ObjectMapper objectMapper = DeltaObjectMapper.getInstance(); @@ -31,33 +27,44 @@ public static BaseOp parseJsonPredicate(String predicate) throws PredicateParsin } } - public static BaseOp parseSqlPredicate(String predicate, DataType dataType) throws JSQLParserException, PredicateException { - var expression = CCJSqlParserUtil.parseCondExpression(predicate); - if (expression instanceof IsNullExpression){ - var isNullExpression = (IsNullExpression) expression; - String column = isNullExpression.getLeftExpression().getASTNode().jjtGetFirstToken().toString(); - var colOp = new ColumnOp(column, dataType); - var children = List.of((LeafOp) colOp); - var operator = "isnull"; - return NonLeafOp.createPartitionFilter(children, operator); - } - else if (expression instanceof BinaryExpression) { - BinaryExpression binaryExpression = (BinaryExpression) expression; - String column = binaryExpression.getLeftExpression().toString(); - String operator = binaryExpression.getStringExpression(); - String value = binaryExpression.getRightExpression().toString(); - var colOp = new ColumnOp(column, dataType); - var litOp = new LiteralOp(value, dataType); - var children = List.of(colOp, litOp); - return NonLeafOp.createPartitionFilter(children, operator); + public static BaseOp parseSqlPredicate(String predicate, EvalContext ctx, Metadata metadata) + throws PredicateException { + try { + var expression = CCJSqlParserUtil.parseCondExpression(predicate); + if (expression instanceof IsNullExpression) { + var isNullExpression = (IsNullExpression) expression; + String column = + isNullExpression.getLeftExpression().getASTNode().jjtGetFirstToken().toString(); + var dataType = metadata.tableSchema().structType().get(column).getDataType(); + var colOp = new ColumnOp(column, dataType); + var children = List.of((LeafOp) colOp); + var operator = "isnull"; + return NonLeafOp.createPartitionFilter(children, operator); + } else if (expression instanceof BinaryExpression) { + BinaryExpression binaryExpression = (BinaryExpression) expression; + String column = binaryExpression.getLeftExpression().toString(); + String operator = binaryExpression.getStringExpression(); + Expression value = binaryExpression.getRightExpression(); + if (value instanceof StringValue) { + StringValue stringValue = (StringValue) value; + var dataType = metadata.tableSchema().structType().get(column).getDataType(); + var colOp = new ColumnOp(column, dataType); + var litOp = new LiteralOp(stringValue.getValue(), dataType); + var children = List.of(colOp, litOp); + return NonLeafOp.createPartitionFilter(children, operator); + } else { + var dataType = metadata.tableSchema().structType().get(column).getDataType(); + var colOp = new ColumnOp(column, dataType); + var litOp = new LiteralOp(value.toString(), dataType); + var children = List.of(colOp, litOp); + return NonLeafOp.createPartitionFilter(children, operator); + } + } else throw new ExpressionNotSupportedException(predicate); + } catch (JSQLParserException e) { + throw new PredicateParsingException(e); } - // TODO: PARSING FAIL on sql; - else - throw new PredicateException(); } - - public static ColumnRange createColumnRange(String name, EvalContext ctx, DataType valueType) throws NonExistingColumnException { var fileStats = ctx.getStatsValues(); diff --git a/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java b/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java index aa6df33ab..7e66593e7 100644 --- a/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java +++ b/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java @@ -7,18 +7,29 @@ public interface ReadTableRequest { - public static class ReadTableVersion implements ReadTableRequest { + class ReadTableVersion implements ReadTableRequest { private final List predicateHints; + private final List jsonPredicateHints; private final Optional limitHint; private final Long version; - public ReadTableVersion(List predicateHints, Optional limitHint, Long version) { + public ReadTableVersion( + List predicateHints, + List jsonPredicateHints, + Optional limitHint, + Long version) { + this.predicateHints = predicateHints; + this.jsonPredicateHints = jsonPredicateHints; this.limitHint = limitHint; this.version = version; } + public List jsonPredicateHints() { + return jsonPredicateHints; + } + public List predicateHints() { return predicateHints; } @@ -38,6 +49,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; ReadTableVersion that = (ReadTableVersion) o; return Objects.equals(predicateHints, that.predicateHints) + && Objects.equals(jsonPredicateHints, that.jsonPredicateHints) && Objects.equals(limitHint, that.limitHint) && Objects.equals(version, that.version); } @@ -45,31 +57,43 @@ public boolean equals(Object o) { @Override @SkipCoverageGenerated public int hashCode() { - return Objects.hash(predicateHints, limitHint, version); + return Objects.hash(predicateHints, jsonPredicateHints, limitHint, version); } @Override @SkipCoverageGenerated public String toString() { return "ReadTableVersion{" + "predicateHints=" - + predicateHints + ", limitHint=" + + predicateHints + "jsonPredicateHints=" + + jsonPredicateHints + ", limitHint=" + limitHint + ", version=" + version + '}'; } } - public static class ReadTableAsOfTimestamp implements ReadTableRequest { + class ReadTableAsOfTimestamp implements ReadTableRequest { private final List predicateHints; + private final Optional limitHint; + private final List jsonPredicateHints; private final Long timestamp; public ReadTableAsOfTimestamp( - List predicateHints, Optional limitHint, Long timestamp) { + List predicateHints, + List jsonPredicateHints, + Optional limitHint, + Long timestamp) { + this.predicateHints = predicateHints; + this.jsonPredicateHints = jsonPredicateHints; this.limitHint = limitHint; this.timestamp = timestamp; } + public List jsonPredicateHints() { + return jsonPredicateHints; + } + @Override @SkipCoverageGenerated public boolean equals(Object o) { @@ -77,6 +101,7 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; ReadTableAsOfTimestamp that = (ReadTableAsOfTimestamp) o; return Objects.equals(predicateHints, that.predicateHints) + && Objects.equals(jsonPredicateHints, that.jsonPredicateHints) && Objects.equals(limitHint, that.limitHint) && Objects.equals(timestamp, that.timestamp); } @@ -84,14 +109,15 @@ public boolean equals(Object o) { @Override @SkipCoverageGenerated public int hashCode() { - return Objects.hash(predicateHints, limitHint, timestamp); + return Objects.hash(jsonPredicateHints, predicateHints, limitHint, timestamp); } @Override @SkipCoverageGenerated public String toString() { return "ReadTableAsOfTimestamp{" + "predicateHints=" - + predicateHints + ", limitHint=" + + predicateHints + "jsonPredicateHints=" + + jsonPredicateHints + ", limitHint=" + limitHint + ", timestamp=" + timestamp + '}'; } @@ -109,12 +135,15 @@ public Long timestamp() { } } - public static class ReadTableCurrentVersion implements ReadTableRequest { + class ReadTableCurrentVersion implements ReadTableRequest { private final List predicateHints; + private final List jsonPredicateHints; private final Optional limitHint; - public ReadTableCurrentVersion(List predicateHints, Optional limitHint) { + public ReadTableCurrentVersion( + List predicateHints, List jsonPredicateHints, Optional limitHint) { this.predicateHints = predicateHints; + this.jsonPredicateHints = jsonPredicateHints; this.limitHint = limitHint; } @@ -122,6 +151,10 @@ public List predicateHints() { return predicateHints; } + public List jsonPredicateHints() { + return jsonPredicateHints; + } + public Optional limitHint() { return limitHint; } @@ -130,7 +163,8 @@ public Optional limitHint() { @SkipCoverageGenerated public String toString() { return "ReadTableCurrentVersion{" + "predicateHints=" - + predicateHints + ", limitHint=" + + predicateHints + "jsonPredicateHints=" + + jsonPredicateHints + ", limitHint=" + limitHint + '}'; } @@ -141,13 +175,14 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; ReadTableCurrentVersion that = (ReadTableCurrentVersion) o; return Objects.equals(predicateHints, that.predicateHints) + && Objects.equals(jsonPredicateHints, that.jsonPredicateHints) && Objects.equals(limitHint, that.limitHint); } @Override @SkipCoverageGenerated public int hashCode() { - return Objects.hash(predicateHints, limitHint); + return Objects.hash(jsonPredicateHints, predicateHints, limitHint); } } } diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index 0ce1240d4..949632c18 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -90,7 +90,7 @@ public Optional getTableVersion(Optional startingTimestamp) { private boolean evaluateJsonPredicate(String predicate, EvalContext ctx, AddFile f) { try { - var parsedPredicate = JsonPredicatesUtils.parseJsonPredicate(predicate); + var parsedPredicate = PredicateUtils.parseJsonPredicate(predicate); return parsedPredicate.evalExpectBoolean(ctx); } catch (PredicateException e) { logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage()); @@ -101,13 +101,44 @@ private boolean evaluateJsonPredicate(String predicate, EvalContext ctx, AddFile } } + private boolean evaluateSqlPredicate( + String predicate, EvalContext ctx, AddFile f, Metadata metadata) { + try { + var parsedPredicate = PredicateUtils.parseSqlPredicate(predicate, ctx, metadata); + return parsedPredicate.evalExpectBoolean(ctx); + } catch (PredicateException e) { + logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage()); + logger.info("File: " + f.getPath() + + " will be used in processing due to failure in parsing or processing the predicate: " + + predicate); + return true; + } + } + + public boolean filterFilesBasedOnSqlPredicates( + List predicates, AddFile f, Metadata metadata) { + // if there are no predicates return all possible files + if (predicates == null) { + return true; + } + try { + var ctx = PredicateUtils.createEvalContext(f); + return predicates.stream().allMatch(p -> evaluateSqlPredicate(p, ctx, f, metadata)); + } catch (PredicateException e) { + logger.debug("Caught exception: " + e.getMessage()); + logger.info("File: " + f.getPath() + + " will be used in processing due to failure in parsing or processing the predicate"); + return true; + } + } + public boolean filterFilesBasedOnJsonPredicates(List predicates, AddFile f) { // if there are no predicates return all possible files if (predicates == null) { return true; } try { - var ctx = JsonPredicatesUtils.createEvalContext(f); + var ctx = PredicateUtils.createEvalContext(f); return predicates.stream().allMatch(p -> evaluateJsonPredicate(p, ctx, f)); } catch (PredicateException e) { logger.debug("Caught exception: " + e.getMessage()); @@ -116,30 +147,38 @@ public boolean filterFilesBasedOnJsonPredicates(List predicates, AddFile return true; } } - ; public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { List predicates; + List sqlPredicates; Snapshot snapshot; if (readTableRequest instanceof ReadTableRequest.ReadTableCurrentVersion) { snapshot = deltaLog.snapshot(); - predicates = ((ReadTableRequest.ReadTableCurrentVersion) readTableRequest).predicateHints(); + predicates = + ((ReadTableRequest.ReadTableCurrentVersion) readTableRequest).jsonPredicateHints(); + sqlPredicates = + ((ReadTableRequest.ReadTableCurrentVersion) readTableRequest).predicateHints(); } else if (readTableRequest instanceof ReadTableRequest.ReadTableAsOfTimestamp) { snapshot = deltaLog.getSnapshotForTimestampAsOf( ((ReadTableRequest.ReadTableAsOfTimestamp) readTableRequest).timestamp()); - predicates = ((ReadTableRequest.ReadTableAsOfTimestamp) readTableRequest).predicateHints(); + predicates = + ((ReadTableRequest.ReadTableAsOfTimestamp) readTableRequest).jsonPredicateHints(); + sqlPredicates = ((ReadTableRequest.ReadTableAsOfTimestamp) readTableRequest).predicateHints(); } else if (readTableRequest instanceof ReadTableRequest.ReadTableVersion) { snapshot = deltaLog.getSnapshotForVersionAsOf( ((ReadTableRequest.ReadTableVersion) readTableRequest).version()); - predicates = ((ReadTableRequest.ReadTableVersion) readTableRequest).predicateHints(); + predicates = ((ReadTableRequest.ReadTableVersion) readTableRequest).jsonPredicateHints(); + sqlPredicates = ((ReadTableRequest.ReadTableVersion) readTableRequest).predicateHints(); } else { throw new IllegalArgumentException("Unknown ReadTableRequest type: " + readTableRequest); } + var metadata = metadataFromSnapshot(snapshot); return new ReadTableResultToBeSigned( new Protocol(Optional.of(1)), - metadataFromSnapshot(snapshot), + metadata, snapshot.getAllFiles().stream() .filter(f -> filterFilesBasedOnJsonPredicates(predicates, f)) + .filter(f -> filterFilesBasedOnSqlPredicates(sqlPredicates, f, metadata)) .map(f -> new TableFileToBeSigned( location() + "/" + f.getPath(), f.getSize(), diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java index 4f5ff41a5..be243b827 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java @@ -1,72 +1,71 @@ package io.whitefox.core.types.predicates; +import static io.whitefox.core.PredicateUtils.createColumnRange; +import static io.whitefox.core.types.predicates.EvaluatorVersion.V1; + import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.whitefox.core.ColumnRange; import io.whitefox.core.types.BooleanType; import io.whitefox.core.types.DataType; -import org.apache.commons.lang3.tuple.Pair; - import java.util.Objects; - -import static io.whitefox.core.JsonPredicatesUtils.createColumnRange; -import static io.whitefox.core.types.predicates.EvaluatorVersion.V1; +import org.apache.commons.lang3.tuple.Pair; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "column") public class ColumnOp extends LeafOp { - @JsonProperty("name") - String name; + @JsonProperty("name") + String name; - public ColumnOp() { - super(); - } + public ColumnOp() { + super(); + } - public ColumnOp(String name, DataType valueType) { - this.name = name; - this.valueType = valueType; - } + public ColumnOp(String name, DataType valueType) { + this.name = name; + this.valueType = valueType; + } - // Determine if the column value is null. - @Override - public Boolean isNull(EvalContext ctx) { - return resolve(ctx) == null; - } + // Determine if the column value is null. + @Override + public Boolean isNull(EvalContext ctx) { + return resolve(ctx) == null; + } - @Override - public Boolean evalExpectBoolean(EvalContext ctx) { - if (!Objects.equals(valueType, BooleanType.BOOLEAN)) { - throw new IllegalArgumentException("Unsupported type for boolean evaluation: " + valueType); - } - return Boolean.valueOf(resolve(ctx)); + @Override + public Boolean evalExpectBoolean(EvalContext ctx) { + if (!Objects.equals(valueType, BooleanType.BOOLEAN)) { + throw new IllegalArgumentException("Unsupported type for boolean evaluation: " + valueType); } + return Boolean.valueOf(resolve(ctx)); + } - public ColumnRange evalExpectColumnRange(EvalContext ctx) throws NonExistingColumnException { - return createColumnRange(name, ctx, valueType); - } + public ColumnRange evalExpectColumnRange(EvalContext ctx) throws NonExistingColumnException { + return createColumnRange(name, ctx, valueType); + } - @Override - public DataType getOpValueType() { - return valueType; - } + @Override + public DataType getOpValueType() { + return valueType; + } - @Override - public Object eval(EvalContext ctx) { - // TODO: handle case of null column + column ranges - return Pair.of(resolve(ctx), valueType); - } + @Override + public Object eval(EvalContext ctx) { + // TODO: handle case of null column + column ranges + return Pair.of(resolve(ctx), valueType); + } - public void validate() throws PredicateException { - if (name == null) { - throw new IllegalArgumentException("Name must be specified: " + this); - } - if (!this.isSupportedType(valueType, V1)) { - throw new TypeNotSupportedException(valueType); - } + public void validate() throws PredicateException { + if (name == null) { + throw new IllegalArgumentException("Name must be specified: " + this); } - - private String resolve(EvalContext ctx) { - // TODO: handle case of null column + column ranges - return ctx.partitionValues.getOrDefault(name, null); + if (!this.isSupportedType(valueType, V1)) { + throw new TypeNotSupportedException(valueType); } + } + + private String resolve(EvalContext ctx) { + // TODO: handle case of null column + column ranges + return ctx.partitionValues.getOrDefault(name, null); + } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index 1f7db4536..769b8e1b0 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -18,8 +18,8 @@ private static LeafEvaluationResult validateAndGetRange( return LeafEvaluationResult.createFromRange(Pair.of(columnRange, rightVal)); } - private static LeafEvaluationResult validateAndGetTypeAndValue(List children, EvalContext ctx) - throws PredicateException { + private static LeafEvaluationResult validateAndGetTypeAndValue( + List children, EvalContext ctx) throws PredicateException { var leftChild = children.get(0); var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); var leftVal = leftChild.evalExpectValueAndType(ctx).getLeft(); @@ -128,6 +128,8 @@ public static void validateValue(String value, DataType valueType) { // TODO check for non deprecated } else if (TimestampType.TIMESTAMP.equals(valueType)) { Timestamp.valueOf(value); + } else if (StringType.STRING.equals(valueType)) { + return; } else { throw new TypeNotSupportedException(valueType); } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/ExpressionNotSupportedException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/ExpressionNotSupportedException.java new file mode 100644 index 000000000..c25526922 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/ExpressionNotSupportedException.java @@ -0,0 +1,14 @@ +package io.whitefox.core.types.predicates; + +public class ExpressionNotSupportedException extends PredicateException { + private final String expression; + + public ExpressionNotSupportedException(String expression) { + this.expression = expression; + } + + @Override + public String getMessage() { + return "Unsupported expression: " + expression.toString(); + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java index a66a937f9..8238928d1 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -4,9 +4,8 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.whitefox.core.types.DataType; -import org.apache.commons.lang3.tuple.Pair; - import java.util.List; +import org.apache.commons.lang3.tuple.Pair; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "op") @JsonSubTypes({ @@ -36,4 +35,3 @@ public List getAllChildren() { abstract DataType getOpValueType(); } - diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LiteralOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LiteralOp.java index fe4d353ce..012c31737 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LiteralOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LiteralOp.java @@ -1,50 +1,49 @@ package io.whitefox.core.types.predicates; +import static io.whitefox.core.types.predicates.EvaluatorVersion.V1; + import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeInfo; import io.whitefox.core.types.DataType; import org.apache.commons.lang3.tuple.Pair; -import static io.whitefox.core.types.predicates.EvaluatorVersion.V1; - @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "literal") -public -class LiteralOp extends LeafOp { - @JsonProperty("value") - String value; - - @Override - public void validate() throws PredicateException { - if (value == null) { - throw new IllegalArgumentException("Value must be specified: " + this); - } - if (!isSupportedType(valueType, V1)) { - throw new IllegalArgumentException("Unsupported type: " + valueType); - } - EvalHelper.validateValue(value, valueType); +public class LiteralOp extends LeafOp { + @JsonProperty("value") + String value; + + @Override + public void validate() throws PredicateException { + if (value == null) { + throw new IllegalArgumentException("Value must be specified: " + this); } - - @Override - public Object eval(EvalContext ctx) { - return Pair.of(value, valueType); - } - - public LiteralOp() { - super(); - } - - public LiteralOp(String value, DataType valueType) { - this.value = value; - this.valueType = valueType; - } - - @Override - public Boolean isNull(EvalContext ctx) { - return false; - } - - @Override - public DataType getOpValueType() { - return valueType; + if (!isSupportedType(valueType, V1)) { + throw new IllegalArgumentException("Unsupported type: " + valueType); } + EvalHelper.validateValue(value, valueType); + } + + @Override + public Object eval(EvalContext ctx) { + return Pair.of(value, valueType); + } + + public LiteralOp() { + super(); + } + + public LiteralOp(String value, DataType valueType) { + this.value = value; + this.valueType = valueType; + } + + @Override + public Boolean isNull(EvalContext ctx) { + return false; + } + + @Override + public DataType getOpValueType() { + return valueType; + } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java index e40655a9a..7931f160a 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java @@ -3,14 +3,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import io.whitefox.core.types.DateType; -import net.sf.jsqlparser.JSQLParserException; -import net.sf.jsqlparser.expression.BinaryExpression; -import net.sf.jsqlparser.parser.CCJSqlParserUtil; -import net.sf.jsqlparser.statement.select.PlainSelect; - import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; // Represents a non-leaf operation. @@ -31,7 +24,8 @@ public abstract class NonLeafOp implements BaseOp { @JsonProperty("children") List children; - public static NonLeafOp createPartitionFilter(List children, String operator) throws PredicateException { + public static NonLeafOp createPartitionFilter(List children, String operator) + throws PredicateException { switch (operator) { case "=": return new EqualOp(children); diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java index 8ece1d34b..9fb890005 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateParsingException.java @@ -1,13 +1,12 @@ package io.whitefox.core.types.predicates; -import com.fasterxml.jackson.core.JsonProcessingException; import java.util.Arrays; public class PredicateParsingException extends PredicateException { - private final JsonProcessingException cause; + private final Exception cause; - public PredicateParsingException(JsonProcessingException cause) { + public PredicateParsingException(Exception cause) { this.cause = cause; } diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index 1dd8047b2..4cd340d33 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -82,7 +82,8 @@ void queryTableWithoutPredicate() { var PTable = new SharedTable( "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); - var request = new ReadTableRequest.ReadTableCurrentVersion(List.of(), Optional.empty()); + var request = + new ReadTableRequest.ReadTableCurrentVersion(List.of(), List.of(), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.protocol(), new Protocol(Optional.of(1))); assertEquals(response.other().size(), 9); @@ -101,12 +102,38 @@ void queryTableWithJsonPredicate() { var PTable = new SharedTable( "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); - var request = - new ReadTableRequest.ReadTableCurrentVersion(List.of(predicate), Optional.empty()); + var request = new ReadTableRequest.ReadTableCurrentVersion( + List.of(), List.of(predicate), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.other().size(), 4); } + @Test + void queryTableWithSqlPredicate() { + var predicate = "date = '2021-08-15'"; + + var PTable = new SharedTable( + "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); + var DTable = DeltaSharedTable.of(PTable); + var request = new ReadTableRequest.ReadTableCurrentVersion( + List.of(predicate), List.of(), Optional.empty()); + var response = DTable.queryTable(request); + assertEquals(4, response.other().size()); + } + + @Test + void queryTableWithNonPartitionSqlPredicate() { + var predicate = "id < 5"; + var tableName = "partitioned-delta-table-with-multiple-columns"; + + var PTable = new SharedTable(tableName, "default", "share1", deltaTable(tableName)); + var DTable = DeltaSharedTable.of(PTable); + var request = new ReadTableRequest.ReadTableCurrentVersion( + List.of(predicate), List.of(), Optional.empty()); + var response = DTable.queryTable(request); + assertEquals(1, response.other().size()); + } + @Test void queryTableWithInvalidJsonPredicate() { var predicate = "{" @@ -120,8 +147,8 @@ void queryTableWithInvalidJsonPredicate() { var PTable = new SharedTable( "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); - var request = - new ReadTableRequest.ReadTableCurrentVersion(List.of(predicate), Optional.empty()); + var request = new ReadTableRequest.ReadTableCurrentVersion( + List.of(), List.of(predicate), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.other().size(), 9); } @@ -139,8 +166,8 @@ void queryTableWithColumnRangePredicate() { var PTable = new SharedTable(tableName, "default", "share1", deltaTable(tableName)); var DTable = DeltaSharedTable.of(PTable); - var request = - new ReadTableRequest.ReadTableCurrentVersion(List.of(predicate), Optional.empty()); + var request = new ReadTableRequest.ReadTableCurrentVersion( + List.of(), List.of(predicate), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.other().size(), 1); } diff --git a/server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java b/server/core/src/test/java/io/whitefox/core/types/PredicateUtilsTest.java similarity index 87% rename from server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java rename to server/core/src/test/java/io/whitefox/core/types/PredicateUtilsTest.java index 14d6180a2..999a0ce4d 100644 --- a/server/core/src/test/java/io/whitefox/core/types/JsonPredicatesUtilsTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/PredicateUtilsTest.java @@ -5,14 +5,14 @@ import io.delta.standalone.DeltaLog; import io.delta.standalone.actions.AddFile; -import io.whitefox.core.JsonPredicatesUtils; +import io.whitefox.core.PredicateUtils; import io.whitefox.core.SharedTable; import io.whitefox.core.types.predicates.*; import java.util.ArrayList; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Test; -public class JsonPredicatesUtilsTest { +public class PredicateUtilsTest { @Test void testCreateEvalContext() throws PredicateParsingException { @@ -26,7 +26,7 @@ void testCreateEvalContext() throws PredicateParsingException { new Configuration(), deltaTableUri("partitioned-delta-table-with-multiple-columns")); var contexts = new ArrayList(); for (AddFile file : log.snapshot().getAllFiles()) { - EvalContext evalContext = JsonPredicatesUtils.createEvalContext(file); + EvalContext evalContext = PredicateUtils.createEvalContext(file); contexts.add(evalContext); } assert (contexts.size() == 2); diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java index 647e6184e..eb8433ff3 100644 --- a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java @@ -2,7 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; -import io.whitefox.core.JsonPredicatesUtils; +import io.whitefox.core.PredicateUtils; import org.junit.jupiter.api.Test; public class PredicateParsingTest { @@ -16,7 +16,7 @@ void testParsingOfEqual() throws PredicateException { + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + " ]\n" + "}"; - var op = JsonPredicatesUtils.parseJsonPredicate(predicate); + var op = PredicateUtils.parseJsonPredicate(predicate); op.validate(); assert (op instanceof EqualOp); assert (((EqualOp) op).children.size() == 2); @@ -41,7 +41,7 @@ void testParsingOfNested() throws PredicateException { + " }\n" + " ]\n" + "}"; - var op = JsonPredicatesUtils.parseJsonPredicate(predicate); + var op = PredicateUtils.parseJsonPredicate(predicate); op.validate(); assert (op instanceof AndOp); assert (((AndOp) op).children.size() == 2); @@ -68,6 +68,6 @@ void testCustomExceptionOnBadJson() { + " ]\n" + "}"; assertThrows( - PredicateParsingException.class, () -> JsonPredicatesUtils.parseJsonPredicate(predicate)); + PredicateParsingException.class, () -> PredicateUtils.parseJsonPredicate(predicate)); } } From 9ad58d937b9b33e1e972d44738282b073c600d18 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Mon, 25 Dec 2023 12:30:57 +0100 Subject: [PATCH 13/30] protocol not compliant --- .../server/DeltaSharesApiImplTest.java | 50 +++++++++++++++++++ .../java/io/whitefox/core/PredicateUtils.java | 31 ++++++++++++ .../core/services/DeltaSharedTable.java | 30 ++--------- .../core/types/predicates/EvalHelper.java | 6 +-- .../predicates/TypeValidationException.java | 19 +++++++ .../predicates/PredicateParsingTest.java | 32 +++++++++++- 6 files changed, 137 insertions(+), 31 deletions(-) create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/TypeValidationException.java diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java index de67b1609..01bb626b4 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java @@ -369,6 +369,56 @@ public void queryNotExistingTable() throws IOException { .statusCode(404); } + @DisabledOnOs(OS.WINDOWS) + @Test + public void queryTableCurrentVersionWithPredicates() throws IOException { + var responseBodyLines = given() + .when() + .filter(deltaFilter) + .body("{\"jsonPredicateHints\": {\"op\":\"and\",\"children\":\"[" + + " {op:not,children:[" + + " {op:isNull\",\"children\":[\n" + + " {\"op\":\"column\",\"name\":\"birthday\",\"valueType\":\"date\"}]}]},\n" + + " {\"op\":\"equal\",\"children\":[\n" + + " {\"op\":\"column\",\"name\":\"birthday\",\"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2020-01-01\",\"valueType\":\"date\"}]}\n" + + "]\"}}") + .header(new Header("Content-Type", "application/json")) + .post( + "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/query", + "name", + "default", + "table1") + .then() + .statusCode(200) + .extract() + .body() + .asString() + .split("\n"); + + assertEquals( + deltaTable1Protocol, + objectMapper.reader().readValue(responseBodyLines[0], ProtocolObject.class)); + assertEquals( + deltaTable1Metadata, + objectMapper.reader().readValue(responseBodyLines[1], MetadataObject.class)); + var files = Arrays.stream(responseBodyLines) + .skip(2) + .map(line -> { + try { + return objectMapper + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .reader() + .readValue(line, FileObjectWithoutPresignedUrl.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toSet()); + assertEquals(7, responseBodyLines.length); + assertEquals(deltaTable1FilesWithoutPresignedUrl, files); // TOD + } + @DisabledOnOs(OS.WINDOWS) @Test public void queryTableCurrentVersion() throws IOException { diff --git a/server/core/src/main/java/io/whitefox/core/PredicateUtils.java b/server/core/src/main/java/io/whitefox/core/PredicateUtils.java index 2e0f6dfb4..1f6e961d1 100644 --- a/server/core/src/main/java/io/whitefox/core/PredicateUtils.java +++ b/server/core/src/main/java/io/whitefox/core/PredicateUtils.java @@ -14,9 +14,12 @@ import net.sf.jsqlparser.expression.operators.relational.IsNullExpression; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import org.apache.commons.lang3.tuple.Pair; +import org.apache.log4j.Logger; public class PredicateUtils { + private static final Logger logger = Logger.getLogger(PredicateUtils.class); + private static final ObjectMapper objectMapper = DeltaObjectMapper.getInstance(); public static BaseOp parseJsonPredicate(String predicate) throws PredicateParsingException { @@ -27,6 +30,34 @@ public static BaseOp parseJsonPredicate(String predicate) throws PredicateParsin } } + + public static boolean evaluateJsonPredicate(String predicate, EvalContext ctx, AddFile f) { + try { + var parsedPredicate = PredicateUtils.parseJsonPredicate(predicate); + return parsedPredicate.evalExpectBoolean(ctx); + } catch (PredicateException e) { + logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage()); + logger.info("File: " + f.getPath() + + " will be used in processing due to failure in parsing or processing the predicate: " + + predicate); + return true; + } + } + + public static boolean evaluateSqlPredicate( + String predicate, EvalContext ctx, AddFile f, Metadata metadata) { + try { + var parsedPredicate = PredicateUtils.parseSqlPredicate(predicate, ctx, metadata); + return parsedPredicate.evalExpectBoolean(ctx); + } catch (PredicateException e) { + logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage()); + logger.info("File: " + f.getPath() + + " will be used in processing due to failure in parsing or processing the predicate: " + + predicate); + return true; + } + } + public static BaseOp parseSqlPredicate(String predicate, EvalContext ctx, Metadata metadata) throws PredicateException { try { diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index 949632c18..5e5aa9873 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -1,5 +1,8 @@ package io.whitefox.core.services; +import static io.whitefox.core.PredicateUtils.evaluateJsonPredicate; +import static io.whitefox.core.PredicateUtils.evaluateSqlPredicate; + import io.delta.standalone.DeltaLog; import io.delta.standalone.Snapshot; import io.delta.standalone.actions.AddFile; @@ -88,33 +91,6 @@ public Optional getTableVersion(Optional startingTimestamp) { return getSnapshot(startingTimestamp).map(Snapshot::getVersion); } - private boolean evaluateJsonPredicate(String predicate, EvalContext ctx, AddFile f) { - try { - var parsedPredicate = PredicateUtils.parseJsonPredicate(predicate); - return parsedPredicate.evalExpectBoolean(ctx); - } catch (PredicateException e) { - logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage()); - logger.info("File: " + f.getPath() - + " will be used in processing due to failure in parsing or processing the predicate: " - + predicate); - return true; - } - } - - private boolean evaluateSqlPredicate( - String predicate, EvalContext ctx, AddFile f, Metadata metadata) { - try { - var parsedPredicate = PredicateUtils.parseSqlPredicate(predicate, ctx, metadata); - return parsedPredicate.evalExpectBoolean(ctx); - } catch (PredicateException e) { - logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage()); - logger.info("File: " + f.getPath() - + " will be used in processing due to failure in parsing or processing the predicate: " - + predicate); - return true; - } - } - public boolean filterFilesBasedOnSqlPredicates( List predicates, AddFile f, Metadata metadata) { // if there are no predicates return all possible files diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index 769b8e1b0..a98a90a54 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -111,7 +111,8 @@ else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { // Validates that the specified value is in the correct format. // Throws an exception otherwise. - public static void validateValue(String value, DataType valueType) { + public static void validateValue(String value, DataType valueType) + throws TypeValidationException { try { if (BooleanType.BOOLEAN.equals(valueType)) { Boolean.parseBoolean(value); @@ -134,8 +135,7 @@ public static void validateValue(String value, DataType valueType) { throw new TypeNotSupportedException(valueType); } } catch (Exception e) { - throw new IllegalArgumentException( - "Error validating " + value + " for type " + valueType + ": " + e); + throw new TypeValidationException(value, valueType); } } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/TypeValidationException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/TypeValidationException.java new file mode 100644 index 000000000..4c96a4e46 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/TypeValidationException.java @@ -0,0 +1,19 @@ +package io.whitefox.core.types.predicates; + +import io.whitefox.core.types.DataType; + +public class TypeValidationException extends PredicateException { + + private final String value; + private final DataType valueType; + + public TypeValidationException(String value, DataType valueType) { + this.value = value; + this.valueType = valueType; + } + + @Override + public String getMessage() { + return "Error validating value: " + value + " for type " + valueType; + } +} diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java index eb8433ff3..1ff0d19d5 100644 --- a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java @@ -2,13 +2,21 @@ import static org.junit.jupiter.api.Assertions.assertThrows; +import io.whitefox.core.Metadata; import io.whitefox.core.PredicateUtils; +import io.whitefox.core.TableSchema; +import io.whitefox.core.types.DateType; +import io.whitefox.core.types.StructField; +import io.whitefox.core.types.StructType; +import java.util.List; +import java.util.Map; +import java.util.Optional; import org.junit.jupiter.api.Test; public class PredicateParsingTest { @Test - void testParsingOfEqual() throws PredicateException { + void testParsingOfJsonEqual() throws PredicateException { var predicate = "{\n" + " \"op\": \"equal\",\n" + " \"children\": [\n" @@ -22,6 +30,28 @@ void testParsingOfEqual() throws PredicateException { assert (((EqualOp) op).children.size() == 2); } + @Test + void testParsingOfSqlEqual() throws PredicateException { + var ctx = new EvalContext(Map.of("date", "date"), Map.of()); + var meta = new Metadata( + "id", + Optional.empty(), + Optional.empty(), + Metadata.Format.PARQUET, + new TableSchema( + new StructType(List.of(new StructField("date", DateType.DATE, true, Map.of())))), + List.of("date", "age"), + Map.of(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + var predicate = "date = '2021-09-09'"; + var op = PredicateUtils.parseSqlPredicate(predicate, ctx, meta); + op.validate(); + assert (op instanceof EqualOp); + assert (((EqualOp) op).children.size() == 2); + } + @Test void testParsingOfNested() throws PredicateException { var predicate = "{\n" + " \"op\":\"and\",\n" From 8d1d4134d85364abb433e74c147204d0757f06e9 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Thu, 28 Dec 2023 16:38:18 +0100 Subject: [PATCH 14/30] final protocol align --- protocol/delta-sharing-protocol-api.yml | 4 +--- .../java/io/whitefox/core/PredicateUtils.java | 11 ++++++---- .../io/whitefox/core/ReadTableRequest.java | 20 ++++++++++--------- .../core/services/DeltaSharedTable.java | 7 +++---- .../core/services/DeltaSharedTableTest.java | 12 +++++------ .../predicates/PredicateParsingTest.java | 6 +++--- 6 files changed, 31 insertions(+), 29 deletions(-) diff --git a/protocol/delta-sharing-protocol-api.yml b/protocol/delta-sharing-protocol-api.yml index 1da49591b..8cd1e94e8 100644 --- a/protocol/delta-sharing-protocol-api.yml +++ b/protocol/delta-sharing-protocol-api.yml @@ -670,7 +670,7 @@ components: items: type: string jsonPredicateHints: - type: array + type: string description: | query predicates on partition columns specified using a structured JSON format. When it’s present, the server will try to use the predicates to filter table's @@ -680,8 +680,6 @@ components: If the server encounters any errors during predicate processing (for example, invalid syntax or non existing columns), it will skip filtering and return all the files. When it’s absent, the server will return all the files in the table. - items: - type: string # properties: # op: # $ref: '#/components/schemas/Ops' diff --git a/server/core/src/main/java/io/whitefox/core/PredicateUtils.java b/server/core/src/main/java/io/whitefox/core/PredicateUtils.java index 1f6e961d1..c1945b971 100644 --- a/server/core/src/main/java/io/whitefox/core/PredicateUtils.java +++ b/server/core/src/main/java/io/whitefox/core/PredicateUtils.java @@ -30,11 +30,14 @@ public static BaseOp parseJsonPredicate(String predicate) throws PredicateParsin } } - - public static boolean evaluateJsonPredicate(String predicate, EvalContext ctx, AddFile f) { + public static boolean evaluateJsonPredicate( + Optional predicate, EvalContext ctx, AddFile f) { try { - var parsedPredicate = PredicateUtils.parseJsonPredicate(predicate); - return parsedPredicate.evalExpectBoolean(ctx); + if (predicate.isEmpty()) return true; + else { + var parsedPredicate = PredicateUtils.parseJsonPredicate(predicate.get()); + return parsedPredicate.evalExpectBoolean(ctx); + } } catch (PredicateException e) { logger.debug("Caught exception for predicate: " + predicate + " - " + e.getMessage()); logger.info("File: " + f.getPath() diff --git a/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java b/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java index 7e66593e7..25fed8d9c 100644 --- a/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java +++ b/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java @@ -9,14 +9,14 @@ public interface ReadTableRequest { class ReadTableVersion implements ReadTableRequest { private final List predicateHints; - private final List jsonPredicateHints; + private final Optional jsonPredicateHints; private final Optional limitHint; private final Long version; public ReadTableVersion( List predicateHints, - List jsonPredicateHints, + Optional jsonPredicateHints, Optional limitHint, Long version) { @@ -26,7 +26,7 @@ public ReadTableVersion( this.version = version; } - public List jsonPredicateHints() { + public Optional jsonPredicateHints() { return jsonPredicateHints; } @@ -75,12 +75,12 @@ class ReadTableAsOfTimestamp implements ReadTableRequest { private final List predicateHints; private final Optional limitHint; - private final List jsonPredicateHints; + private final Optional jsonPredicateHints; private final Long timestamp; public ReadTableAsOfTimestamp( List predicateHints, - List jsonPredicateHints, + Optional jsonPredicateHints, Optional limitHint, Long timestamp) { @@ -90,7 +90,7 @@ public ReadTableAsOfTimestamp( this.timestamp = timestamp; } - public List jsonPredicateHints() { + public Optional jsonPredicateHints() { return jsonPredicateHints; } @@ -137,11 +137,13 @@ public Long timestamp() { class ReadTableCurrentVersion implements ReadTableRequest { private final List predicateHints; - private final List jsonPredicateHints; + private final Optional jsonPredicateHints; private final Optional limitHint; public ReadTableCurrentVersion( - List predicateHints, List jsonPredicateHints, Optional limitHint) { + List predicateHints, + Optional jsonPredicateHints, + Optional limitHint) { this.predicateHints = predicateHints; this.jsonPredicateHints = jsonPredicateHints; this.limitHint = limitHint; @@ -151,7 +153,7 @@ public List predicateHints() { return predicateHints; } - public List jsonPredicateHints() { + public Optional jsonPredicateHints() { return jsonPredicateHints; } diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index 5e5aa9873..cf18fa54d 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -9,7 +9,6 @@ import io.whitefox.core.*; import io.whitefox.core.Metadata; import io.whitefox.core.TableSchema; -import io.whitefox.core.types.predicates.EvalContext; import io.whitefox.core.types.predicates.PredicateException; import java.sql.Timestamp; import java.time.OffsetDateTime; @@ -108,14 +107,14 @@ public boolean filterFilesBasedOnSqlPredicates( } } - public boolean filterFilesBasedOnJsonPredicates(List predicates, AddFile f) { + public boolean filterFilesBasedOnJsonPredicates(Optional predicates, AddFile f) { // if there are no predicates return all possible files if (predicates == null) { return true; } try { var ctx = PredicateUtils.createEvalContext(f); - return predicates.stream().allMatch(p -> evaluateJsonPredicate(p, ctx, f)); + return evaluateJsonPredicate(predicates, ctx, f); } catch (PredicateException e) { logger.debug("Caught exception: " + e.getMessage()); logger.info("File: " + f.getPath() @@ -125,7 +124,7 @@ public boolean filterFilesBasedOnJsonPredicates(List predicates, AddFile } public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { - List predicates; + Optional predicates; List sqlPredicates; Snapshot snapshot; if (readTableRequest instanceof ReadTableRequest.ReadTableCurrentVersion) { diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index 4cd340d33..b0d5b0bff 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -83,7 +83,7 @@ void queryTableWithoutPredicate() { "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); var request = - new ReadTableRequest.ReadTableCurrentVersion(List.of(), List.of(), Optional.empty()); + new ReadTableRequest.ReadTableCurrentVersion(List.of(), Optional.empty(), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.protocol(), new Protocol(Optional.of(1))); assertEquals(response.other().size(), 9); @@ -103,7 +103,7 @@ void queryTableWithJsonPredicate() { "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(), List.of(predicate), Optional.empty()); + List.of(), Optional.of(predicate), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.other().size(), 4); } @@ -116,7 +116,7 @@ void queryTableWithSqlPredicate() { "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(predicate), List.of(), Optional.empty()); + List.of(predicate), Optional.empty(), Optional.empty()); var response = DTable.queryTable(request); assertEquals(4, response.other().size()); } @@ -129,7 +129,7 @@ void queryTableWithNonPartitionSqlPredicate() { var PTable = new SharedTable(tableName, "default", "share1", deltaTable(tableName)); var DTable = DeltaSharedTable.of(PTable); var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(predicate), List.of(), Optional.empty()); + List.of(predicate), Optional.empty(), Optional.empty()); var response = DTable.queryTable(request); assertEquals(1, response.other().size()); } @@ -148,7 +148,7 @@ void queryTableWithInvalidJsonPredicate() { "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(), List.of(predicate), Optional.empty()); + List.of(), Optional.of(predicate), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.other().size(), 9); } @@ -167,7 +167,7 @@ void queryTableWithColumnRangePredicate() { var PTable = new SharedTable(tableName, "default", "share1", deltaTable(tableName)); var DTable = DeltaSharedTable.of(PTable); var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(), List.of(predicate), Optional.empty()); + List.of(), Optional.of(predicate), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.other().size(), 1); } diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java index 1ff0d19d5..88025f463 100644 --- a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java @@ -18,7 +18,7 @@ public class PredicateParsingTest { @Test void testParsingOfJsonEqual() throws PredicateException { - var predicate = "{\n" + " \"op\": \"equal\",\n" + String predicate = "{\n" + " \"op\": \"equal\",\n" + " \"children\": [\n" + " {\"op\": \"column\", \"name\":\"hireDate\", \"valueType\":\"date\"},\n" + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" @@ -54,7 +54,7 @@ void testParsingOfSqlEqual() throws PredicateException { @Test void testParsingOfNested() throws PredicateException { - var predicate = "{\n" + " \"op\":\"and\",\n" + String predicate = "{\n" + " \"op\":\"and\",\n" + " \"children\":[\n" + " {\n" + " \"op\":\"equal\",\n" @@ -80,7 +80,7 @@ void testParsingOfNested() throws PredicateException { @Test void testCustomExceptionOnBadJson() { - var predicate = "{\n" + " \"op\":\"and\",\n" + String predicate = "{\n" + " \"op\":\"and\",\n" + " \"children\":[\n" + " {\n" + " \"op\":\"equals\",\n" From 03ebecf9ecc5d6f63fde3293292d99399d675950 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 5 Jan 2024 14:53:13 +0100 Subject: [PATCH 15/30] fixed and added tests --- .../api/deltasharing/DeltaMappers.java | 12 +- .../io/whitefox/core/ReadTableRequest.java | 18 +-- .../core/services/DeltaSharedTable.java | 10 +- .../core/types/predicates/BaseOp.java | 2 +- .../core/types/predicates/ColumnOp.java | 2 +- .../core/types/predicates/NonLeafOp.java | 2 +- .../core/services/DeltaSharedTableTest.java | 132 ++++++++++++++---- .../_delta_log/.00000000000000000002.json.crc | Bin 0 -> 24 bytes .../_delta_log/00000000000000000002.json | 5 + ...-b3e0-9d4ea6c178d9.c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...4742-b3e0-9d4ea6c178d9.c000.snappy.parquet | Bin 0 -> 1229 bytes ...-9b7d-01a49d5e5e9a.c000.snappy.parquet.crc | Bin 0 -> 20 bytes ...4ae4-9b7d-01a49d5e5e9a.c000.snappy.parquet | Bin 0 -> 1275 bytes 13 files changed, 136 insertions(+), 47 deletions(-) create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/.00000000000000000002.json.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/00000000000000000002.json create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-09-12/.part-00000-78e140ba-2b39-4742-b3e0-9d4ea6c178d9.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-09-12/part-00000-78e140ba-2b39-4742-b3e0-9d4ea6c178d9.c000.snappy.parquet create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2022-02-06/.part-00001-09dfde6d-8885-4ae4-9b7d-01a49d5e5e9a.c000.snappy.parquet.crc create mode 100644 server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2022-02-06/part-00001-09dfde6d-8885-4ae4-9b7d-01a49d5e5e9a.c000.snappy.parquet diff --git a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java index f85f5bba3..6d52609f4 100644 --- a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java +++ b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java @@ -33,20 +33,20 @@ public static ReadTableRequest api2ReadTableRequest(QueryRequest request) { throw new IllegalArgumentException("version cannot be negative."); } else if (request.getVersion() != null && request.getTimestamp() == null) { return new ReadTableRequest.ReadTableVersion( - request.getPredicateHints(), - request.getJsonPredicateHints(), + Optional.ofNullable(request.getPredicateHints()), + Optional.ofNullable(request.getJsonPredicateHints()), Optional.ofNullable(request.getLimitHint()), request.getVersion()); } else if (request.getVersion() == null && request.getTimestamp() != null) { return new ReadTableRequest.ReadTableAsOfTimestamp( - request.getPredicateHints(), - request.getJsonPredicateHints(), + Optional.ofNullable(request.getPredicateHints()), + Optional.ofNullable(request.getJsonPredicateHints()), Optional.ofNullable(request.getLimitHint()), CommonMappers.parseTimestamp(request.getTimestamp())); } else if (request.getVersion() == null && request.getTimestamp() == null) { return new ReadTableRequest.ReadTableCurrentVersion( - request.getPredicateHints(), - request.getJsonPredicateHints(), + Optional.ofNullable(request.getPredicateHints()), + Optional.ofNullable(request.getJsonPredicateHints()), Optional.ofNullable(request.getLimitHint())); } else { throw new IllegalArgumentException("Cannot specify both version and timestamp"); diff --git a/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java b/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java index 25fed8d9c..55950d454 100644 --- a/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java +++ b/server/core/src/main/java/io/whitefox/core/ReadTableRequest.java @@ -8,14 +8,14 @@ public interface ReadTableRequest { class ReadTableVersion implements ReadTableRequest { - private final List predicateHints; + private final Optional> predicateHints; private final Optional jsonPredicateHints; private final Optional limitHint; private final Long version; public ReadTableVersion( - List predicateHints, + Optional> predicateHints, Optional jsonPredicateHints, Optional limitHint, Long version) { @@ -30,7 +30,7 @@ public Optional jsonPredicateHints() { return jsonPredicateHints; } - public List predicateHints() { + public Optional> predicateHints() { return predicateHints; } @@ -72,14 +72,14 @@ public String toString() { } class ReadTableAsOfTimestamp implements ReadTableRequest { - private final List predicateHints; + private final Optional> predicateHints; private final Optional limitHint; private final Optional jsonPredicateHints; private final Long timestamp; public ReadTableAsOfTimestamp( - List predicateHints, + Optional> predicateHints, Optional jsonPredicateHints, Optional limitHint, Long timestamp) { @@ -122,7 +122,7 @@ public String toString() { + timestamp + '}'; } - public List predicateHints() { + public Optional> predicateHints() { return predicateHints; } @@ -136,12 +136,12 @@ public Long timestamp() { } class ReadTableCurrentVersion implements ReadTableRequest { - private final List predicateHints; + private final Optional> predicateHints; private final Optional jsonPredicateHints; private final Optional limitHint; public ReadTableCurrentVersion( - List predicateHints, + Optional> predicateHints, Optional jsonPredicateHints, Optional limitHint) { this.predicateHints = predicateHints; @@ -149,7 +149,7 @@ public ReadTableCurrentVersion( this.limitHint = limitHint; } - public List predicateHints() { + public Optional> predicateHints() { return predicateHints; } diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index cf18fa54d..826a36a1f 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -91,14 +91,14 @@ public Optional getTableVersion(Optional startingTimestamp) { } public boolean filterFilesBasedOnSqlPredicates( - List predicates, AddFile f, Metadata metadata) { + Optional> predicates, AddFile f, Metadata metadata) { // if there are no predicates return all possible files - if (predicates == null) { + if (predicates.isEmpty()) { return true; } try { var ctx = PredicateUtils.createEvalContext(f); - return predicates.stream().allMatch(p -> evaluateSqlPredicate(p, ctx, f, metadata)); + return predicates.get().stream().allMatch(p -> evaluateSqlPredicate(p, ctx, f, metadata)); } catch (PredicateException e) { logger.debug("Caught exception: " + e.getMessage()); logger.info("File: " + f.getPath() @@ -109,7 +109,7 @@ public boolean filterFilesBasedOnSqlPredicates( public boolean filterFilesBasedOnJsonPredicates(Optional predicates, AddFile f) { // if there are no predicates return all possible files - if (predicates == null) { + if (predicates.isEmpty()) { return true; } try { @@ -125,7 +125,7 @@ public boolean filterFilesBasedOnJsonPredicates(Optional predicates, Add public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) { Optional predicates; - List sqlPredicates; + Optional> sqlPredicates; Snapshot snapshot; if (readTableRequest instanceof ReadTableRequest.ReadTableCurrentVersion) { snapshot = deltaLog.snapshot(); diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java index f229ba67d..9e84f22ac 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java @@ -18,7 +18,7 @@ @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), - @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqualOp") + @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqual") }) public interface BaseOp { diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java index be243b827..dc4f3dc54 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java @@ -29,7 +29,7 @@ public ColumnOp(String name, DataType valueType) { // Determine if the column value is null. @Override public Boolean isNull(EvalContext ctx) { - return resolve(ctx) == null; + return ctx.partitionValues.get(name) == null && !ctx.getStatsValues().containsKey(name); } @Override diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java index 7931f160a..2d2611cd8 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java @@ -17,7 +17,7 @@ @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), - @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqualOp") + @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqual") }) public abstract class NonLeafOp implements BaseOp { diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index b0d5b0bff..5a6b34cd3 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -14,6 +14,7 @@ import java.util.List; import java.util.Optional; import java.util.concurrent.ExecutionException; +import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledOnOs; import org.junit.jupiter.api.condition.OS; @@ -82,8 +83,8 @@ void queryTableWithoutPredicate() { var PTable = new SharedTable( "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); - var request = - new ReadTableRequest.ReadTableCurrentVersion(List.of(), Optional.empty(), Optional.empty()); + var request = new ReadTableRequest.ReadTableCurrentVersion( + Optional.empty(), Optional.empty(), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.protocol(), new Protocol(Optional.of(1))); assertEquals(response.other().size(), 9); @@ -103,54 +104,137 @@ void queryTableWithJsonPredicate() { "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(), Optional.of(predicate), Optional.empty()); + Optional.empty(), Optional.of(predicate), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.other().size(), 4); } @Test - void queryTableWithSqlPredicate() { - var predicate = "date = '2021-08-15'"; + void queryTableWithSqlPredicates() { + + var predicatesAndExpectedResult = List.of( + Pair.of(List.of("date = '2021-08-15'"), 4), + Pair.of(List.of("date < '2021-08-14'"), 5), + Pair.of(List.of("date > '2021-08-04'"), 9), + Pair.of(List.of("date is NULL"), 0), + Pair.of(List.of("date >= '2021-08-15'"), 4), + Pair.of(List.of("date <= '2021-08-15'"), 9)); var PTable = new SharedTable( "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); var DTable = DeltaSharedTable.of(PTable); - var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(predicate), Optional.empty(), Optional.empty()); - var response = DTable.queryTable(request); - assertEquals(4, response.other().size()); + + predicatesAndExpectedResult.forEach(p -> { + var request = new ReadTableRequest.ReadTableCurrentVersion( + Optional.of(p.getLeft()), Optional.empty(), Optional.empty()); + var response = DTable.queryTable(request); + assertEquals(p.getRight(), response.other().size()); + }); } @Test void queryTableWithNonPartitionSqlPredicate() { - var predicate = "id < 5"; + var predicates = List.of("id < 5"); var tableName = "partitioned-delta-table-with-multiple-columns"; var PTable = new SharedTable(tableName, "default", "share1", deltaTable(tableName)); var DTable = DeltaSharedTable.of(PTable); var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(predicate), Optional.empty(), Optional.empty()); + Optional.of(predicates), Optional.empty(), Optional.empty()); var response = DTable.queryTable(request); assertEquals(1, response.other().size()); } @Test void queryTableWithInvalidJsonPredicate() { - var predicate = "{" - + " \"op\":\"equal\",\n" - + " \"children\":[\n" - + " {\"op\":\"column\",\"name\":\"dating\",\"valueType\":\"date\"},\n" - + " {\"op\":\"literal\",\"value\":\"2021-08-15\",\"valueType\":\"date\"}\n" - + " ]\n" - + "}"; + + var predicatesAndExpectedResult = List.of( + Pair.of( + "{" + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"dating\",\"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-09-12\",\"valueType\":\"date\"}\n" + + " ]\n" + + "}", + 2), + Pair.of( + "{" + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"date\",\"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-09-12\",\"valueType\":\"date\"}\n" + + " ]\n" + + "}", + 1), + Pair.of( + "{" + + " \"op\":\"lessThan\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + + " {\"op\":\"literal\",\"value\":\"31\",\"valueType\":\"int\"}\n" + + " ]\n" + + "}", + 1), + Pair.of( + "{" + + " \"op\":\"lessThanOrEqual\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + + " {\"op\":\"literal\",\"value\":\"31\",\"valueType\":\"int\"}\n" + + " ]\n" + + "}", + 2), + Pair.of( + "{" + + " \"op\":\"lessThan\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + + " {\"op\":\"literal\",\"value\":\"45\",\"valueType\":\"int\"}\n" + + " ]\n" + + "}", + 2), + Pair.of( + "{" + + " \"op\":\"isNull\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"}" + + " ]\n" + + "}", + 2), + Pair.of( + "{\n" + " \"op\":\"and\",\n" + + " \"children\":[\n" + + " {\n" + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"date\",\"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2022-02-06\",\"valueType\":\"date\"}\n" + + " ]\n" + + " },\n" + + " {\n" + + " \"op\":\"greaterThanOrEqual\",\"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + + " {\"op\":\"literal\",\"value\":\"31\",\"valueType\":\"int\"}\n" + + " ]\n" + + " }\n" + + " ]\n" + + "}", + 0)); var PTable = new SharedTable( - "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); + "partitioned-delta-table-with-multiple-columns", + "default", + "share1", + deltaTable("partitioned-delta-table-with-multiple-columns")); var DTable = DeltaSharedTable.of(PTable); - var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(), Optional.of(predicate), Optional.empty()); - var response = DTable.queryTable(request); - assertEquals(response.other().size(), 9); + + predicatesAndExpectedResult.forEach(p -> { + var request = new ReadTableRequest.ReadTableCurrentVersion( + Optional.empty(), Optional.of(p.getLeft()), Optional.empty()); + var response = DTable.queryTable(request); + assertEquals(p.getRight(), response.other().size()); + }); } @Test @@ -167,7 +251,7 @@ void queryTableWithColumnRangePredicate() { var PTable = new SharedTable(tableName, "default", "share1", deltaTable(tableName)); var DTable = DeltaSharedTable.of(PTable); var request = new ReadTableRequest.ReadTableCurrentVersion( - List.of(), Optional.of(predicate), Optional.empty()); + Optional.empty(), Optional.of(predicate), Optional.empty()); var response = DTable.queryTable(request); assertEquals(response.other().size(), 1); } diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/.00000000000000000002.json.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/.00000000000000000002.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..582d4ecf2922d18b0294d7b5d0940ca80a58d0cc GIT binary patch literal 24 fcmYc;N@ieSU}AV5v!HaZ*|~iV`O>B4Om%z!V-W}* literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/00000000000000000002.json b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/00000000000000000002.json new file mode 100644 index 000000000..d8e84daa2 --- /dev/null +++ b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/_delta_log/00000000000000000002.json @@ -0,0 +1,5 @@ +{"commitInfo":{"timestamp":1704459098200,"operation":"WRITE","operationParameters":{"mode":"Overwrite","partitionBy":"[\"date\"]"},"readVersion":1,"isolationLevel":"Serializable","isBlindAppend":false,"operationMetrics":{"numFiles":"2","numOutputRows":"19","numOutputBytes":"2504"},"engineInfo":"Apache-Spark/3.3.0 Delta-Lake/2.3.0","txnId":"0775569b-f4aa-445e-bd97-31ad2f713bac"}} +{"add":{"path":"date=2021-09-12/part-00000-78e140ba-2b39-4742-b3e0-9d4ea6c178d9.c000.snappy.parquet","partitionValues":{"date":"2021-09-12"},"size":1229,"modificationTime":1704459094688,"dataChange":true,"stats":"{\"numRecords\":9,\"minValues\":{\"id\":31,\"uid\":\"042b58d9-f526-4b1e-a712-1d2b62c8\"},\"maxValues\":{\"id\":39,\"uid\":\"f7af65ac-289c-4142-8fc8-5ce6e3c4�\"},\"nullCount\":{\"id\":0,\"uid\":0}}"}} +{"add":{"path":"date=2022-02-06/part-00001-09dfde6d-8885-4ae4-9b7d-01a49d5e5e9a.c000.snappy.parquet","partitionValues":{"date":"2022-02-06"},"size":1275,"modificationTime":1704459094688,"dataChange":true,"stats":"{\"numRecords\":10,\"minValues\":{\"id\":20,\"uid\":\"06215b25-a3c6-4d69-b000-3487dc0b\"},\"maxValues\":{\"id\":29,\"uid\":\"bd2322fe-fc6c-4417-a3f9-e55b1a53�\"},\"nullCount\":{\"id\":0,\"uid\":0}}"}} +{"remove":{"path":"date=2021-08-09/part-00000-1dcbbcb3-4245-4f20-86d8-01231ba570e7.c000.snappy.parquet","deletionTimestamp":1704459098199,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"date":"2021-08-09"},"size":1044}} +{"remove":{"path":"date=2021-08-15/part-00001-f38b954d-41f3-41c3-a7a7-3115e28814b7.c000.snappy.parquet","deletionTimestamp":1704459098199,"dataChange":true,"extendedFileMetadata":true,"partitionValues":{"date":"2021-08-15"},"size":1003}} diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-09-12/.part-00000-78e140ba-2b39-4742-b3e0-9d4ea6c178d9.c000.snappy.parquet.crc b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-09-12/.part-00000-78e140ba-2b39-4742-b3e0-9d4ea6c178d9.c000.snappy.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..dc3b35723f7b7a132737adc7327c52199910ad57 GIT binary patch literal 20 ccmYc;N@ieSU}AXvvhkQy+VVxM=hdzN07$L~Z2$lO literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-09-12/part-00000-78e140ba-2b39-4742-b3e0-9d4ea6c178d9.c000.snappy.parquet b/server/core/src/testFixtures/resources/delta/samples/partitioned-delta-table-with-multiple-columns/date=2021-09-12/part-00000-78e140ba-2b39-4742-b3e0-9d4ea6c178d9.c000.snappy.parquet new file mode 100644 index 0000000000000000000000000000000000000000..cb167afdbef1874b8c6c042e6794ed1f69eec952 GIT binary patch literal 1229 zcma)6OKTiQ5bjw^yR#r4vZRL@SnzrxZDiOUrXSPOBSav#V1f~WoRSdw=`2}2ybn7; z=$LCR_R07X&^hGfg#3p50+UlNz8PJTOD?IA6v>c7)X+?I*H>RveO=7nho4Z4@QVn~ z@XH_GyuLNW12}hajL?H&2O%_w?x22zM*ZFzx_diDIqdgfZ^GWX9q*0-)2{a~J_w?J zU&nu}R=Zp3^bzq*8&{+G`@b zQp8BANE=P9s2WrH7AKst#sG^dbxnlOCDBp_LM07#qZs9-&EDx`X?zFJ?E>@ai8@&V z7p;5qCi(*GCi3D7+F=sIJV64K4U^LdKkK$OICnqVSRCp3*IzE|SAJjPHH6o;I%#L> z5Z(ae)@tgu(8ohSCxdL5O~&yMuVqyi#aWBOon(;w79}sDalCuo&2@_j+P>=X`WDK7 znS=s>_Oy?aOzoZ!@2>f8R}>b-}eO(e{mczL|h;m+)&Jr^#PE{9}ZY zSG{B~j+38y?}Av9G(DU@EcB5Ed5UwTipPGwm>wRa>%8DaiPC%jK`b843J~?t$}fxg z;ge$DFSXN4oj#3@^X0ckKHtd~%lXPK^GP12e&!bW&S%H@f!=RjQwP_Vu-W0kLpU9* zW;1PO;DsxynnErq87bhsae_1TqpN}rP8_|NYf{nJ#YC3bH sy7n-3hT4$`OX+{>G}Nhg%aZs z|2f2ezj*cf#(R`cue>=rCA|qy+Huxe-V!5RN2KpGv6NCGr0Tt+)_Y46js@#f#~cyL zF(O+=iDA4afyWpv)z0-U;S^I&ctMG7J3(Xwh*d!n9aIdW^@4Ra+j{$PW=$`s@RER6 zCPGJSbdES7+Ne0S!i7ZGGh>8^5H9o{2vs69oOG@0tgyyuBNNVgE;x^YMAtbYCF`N2 zs7YvB%S;xW7su#4dJz0gK{wS2(QED%4+Iwp|8{E6{BigO*Ub6*Q$1i@+zV5pcoaevf^2` zneSXwb5UY}w$5w3utFLzy^sLVKD>sD;`zEFR1n5cR<-ESvfMFU@XP8gG`Se3Bj3%U=&dynY}`u~^j9ij5hS+Znz zzHwIF25T5^_{Nr9O=o_i?}MODQ7we(f@`@GE}G5>Y5Nd`;0(S@r{(Qx0;7_q<$GxD IUHI+212pd*bN~PV literal 0 HcmV?d00001 From 5a63e34f0bdaf8fb2a5d50382f0ca08ca2874d18 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 5 Jan 2024 15:58:58 +0100 Subject: [PATCH 16/30] fix test v2 --- .../server/DeltaSharesApiImplTest.java | 15 +++++++-------- .../core/services/DeltaSharedTableTest.java | 2 +- .../whitefox/core/types/PredicateUtilsTest.java | 4 +++- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java index 01bb626b4..e8908ae0c 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java @@ -375,14 +375,13 @@ public void queryTableCurrentVersionWithPredicates() throws IOException { var responseBodyLines = given() .when() .filter(deltaFilter) - .body("{\"jsonPredicateHints\": {\"op\":\"and\",\"children\":\"[" + - " {op:not,children:[" + - " {op:isNull\",\"children\":[\n" + - " {\"op\":\"column\",\"name\":\"birthday\",\"valueType\":\"date\"}]}]},\n" + - " {\"op\":\"equal\",\"children\":[\n" + - " {\"op\":\"column\",\"name\":\"birthday\",\"valueType\":\"date\"},\n" + - " {\"op\":\"literal\",\"value\":\"2020-01-01\",\"valueType\":\"date\"}]}\n" + - "]\"}}") + .body("{\"jsonPredicateHints\": {\n" + + " \"op\": \"equal\",\n" + + " \"children\": [\n" + + " {\"op\": \"column\", \"name\":\"date\", \"valueType\":\"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + + " ]\n" + + "}}") .header(new Header("Content-Type", "application/json")) .post( "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/query", diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index 5a6b34cd3..a949f5ca3 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -134,7 +134,7 @@ void queryTableWithSqlPredicates() { @Test void queryTableWithNonPartitionSqlPredicate() { - var predicates = List.of("id < 5"); + var predicates = List.of("id < 30"); var tableName = "partitioned-delta-table-with-multiple-columns"; var PTable = new SharedTable(tableName, "default", "share1", deltaTable(tableName)); diff --git a/server/core/src/test/java/io/whitefox/core/types/PredicateUtilsTest.java b/server/core/src/test/java/io/whitefox/core/types/PredicateUtilsTest.java index 999a0ce4d..8c21e9d8d 100644 --- a/server/core/src/test/java/io/whitefox/core/types/PredicateUtilsTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/PredicateUtilsTest.java @@ -31,6 +31,8 @@ void testCreateEvalContext() throws PredicateParsingException { } assert (contexts.size() == 2); var c1 = contexts.get(0); - assert (c1.getPartitionValues().get("date").equals("2021-08-09")); + var c2 = contexts.get(1); + assert (c1.getPartitionValues().get("date").equals("2022-02-06")); + assert (c2.getPartitionValues().get("date").equals("2021-09-12")); } } From 58ea7bdb71cc2be7fc2afcea876b2f2abb532756 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 5 Jan 2024 16:11:17 +0100 Subject: [PATCH 17/30] fixed integration test --- .../api/deltasharing/server/DeltaSharesApiImplTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java index e8908ae0c..21d16fc51 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java @@ -375,13 +375,13 @@ public void queryTableCurrentVersionWithPredicates() throws IOException { var responseBodyLines = given() .when() .filter(deltaFilter) - .body("{\"jsonPredicateHints\": {\n" + + .body("{\"jsonPredicateHints\": \"{\n" + " \"op\": \"equal\",\n" + " \"children\": [\n" + " {\"op\": \"column\", \"name\":\"date\", \"valueType\":\"date\"},\n" + " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + " ]\n" + - "}}") + "}\"}") .header(new Header("Content-Type", "application/json")) .post( "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/query", From 125cef74b3f66410c7d4a96aa8435792a97f89f4 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 5 Jan 2024 16:21:22 +0100 Subject: [PATCH 18/30] correct json body --- .../deltasharing/server/DeltaSharesApiImplTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java index 21d16fc51..370f456a6 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java @@ -375,12 +375,12 @@ public void queryTableCurrentVersionWithPredicates() throws IOException { var responseBodyLines = given() .when() .filter(deltaFilter) - .body("{\"jsonPredicateHints\": \"{\n" + - " \"op\": \"equal\",\n" + - " \"children\": [\n" + - " {\"op\": \"column\", \"name\":\"date\", \"valueType\":\"date\"},\n" + - " {\"op\":\"literal\",\"value\":\"2021-04-29\",\"valueType\":\"date\"}\n" + - " ]\n" + + .body("{\"jsonPredicateHints\": \"{" + + " \\\"op\\\": \\\"equal\\\"," + + " \\\"children\\\": [" + + " {\\\"op\\\": \\\"column\\\", \\\"name\\\":\\\"date\\\", \\\"valueType\\\":\\\"date\\\"}," + + " {\\\"op\\\":\\\"literal\\\",\\\"value\\\":\\\"2021-04-29\\\",\\\"valueType\\\":\\\"date\\\"}" + + " ]" + "}\"}") .header(new Header("Content-Type", "application/json")) .post( From 4cf584b6f2f99d54b35528f7d68fa31b5f632252 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 5 Jan 2024 16:26:16 +0100 Subject: [PATCH 19/30] removed not needed test --- .../core/services/DeltaSharedTableTest.java | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index a949f5ca3..50f8441e6 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -236,23 +236,4 @@ void queryTableWithInvalidJsonPredicate() { assertEquals(p.getRight(), response.other().size()); }); } - - @Test - void queryTableWithColumnRangePredicate() { - var tableName = "partitioned-delta-table-with-multiple-columns"; - var predicate = "{" - + " \"op\":\"lessThan\",\n" - + " \"children\":[\n" - + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" - + " {\"op\":\"literal\",\"value\":\"4\",\"valueType\":\"int\"}\n" - + " ]\n" - + "}"; - - var PTable = new SharedTable(tableName, "default", "share1", deltaTable(tableName)); - var DTable = DeltaSharedTable.of(PTable); - var request = new ReadTableRequest.ReadTableCurrentVersion( - Optional.empty(), Optional.of(predicate), Optional.empty()); - var response = DTable.queryTable(request); - assertEquals(response.other().size(), 1); - } } From bdcd60b10eef14501750f97e42010ef19beaceb3 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 5 Jan 2024 20:14:20 +0100 Subject: [PATCH 20/30] exception tests --- .../whitefox.java-conventions.gradle.kts | 2 +- .../api/deltasharing/DeltaMappers.java | 2 +- .../server/DeltaSharesApiImplTest.java | 73 +++++++++-------- .../core/types/predicates/BaseOp.java | 6 +- .../core/types/predicates/ColumnOp.java | 2 +- .../core/types/predicates/NonLeafOp.java | 10 +-- .../predicates/TypeMismatchException.java | 2 +- .../core/services/DeltaSharedTableTest.java | 21 +---- .../predicates/PredicateExceptionsTest.java | 81 +++++++++++++++++++ .../predicates/PredicateParsingTest.java | 23 ++++++ 10 files changed, 150 insertions(+), 72 deletions(-) create mode 100644 server/core/src/test/java/io/whitefox/core/types/predicates/PredicateExceptionsTest.java diff --git a/buildSrc/src/main/kotlin/whitefox.java-conventions.gradle.kts b/buildSrc/src/main/kotlin/whitefox.java-conventions.gradle.kts index 869d2e156..700052872 100644 --- a/buildSrc/src/main/kotlin/whitefox.java-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/whitefox.java-conventions.gradle.kts @@ -34,7 +34,7 @@ tasks.withType().configureEach { tasks.register("devCheck") { dependsOn(tasks.spotlessApply) - finalizedBy(tasks.check) + finalizedBy(tasks.test) description = "Useful command when iterating locally to apply spotless formatting then running all the checks" } diff --git a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java index 6d52609f4..47503f78e 100644 --- a/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java +++ b/server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java @@ -39,7 +39,7 @@ public static ReadTableRequest api2ReadTableRequest(QueryRequest request) { request.getVersion()); } else if (request.getVersion() == null && request.getTimestamp() != null) { return new ReadTableRequest.ReadTableAsOfTimestamp( - Optional.ofNullable(request.getPredicateHints()), + Optional.ofNullable(request.getPredicateHints()), Optional.ofNullable(request.getJsonPredicateHints()), Optional.ofNullable(request.getLimitHint()), CommonMappers.parseTimestamp(request.getTimestamp())); diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java index 370f456a6..e24bf0b20 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java @@ -373,47 +373,46 @@ public void queryNotExistingTable() throws IOException { @Test public void queryTableCurrentVersionWithPredicates() throws IOException { var responseBodyLines = given() - .when() - .filter(deltaFilter) - .body("{\"jsonPredicateHints\": \"{" + - " \\\"op\\\": \\\"equal\\\"," + - " \\\"children\\\": [" + - " {\\\"op\\\": \\\"column\\\", \\\"name\\\":\\\"date\\\", \\\"valueType\\\":\\\"date\\\"}," + - " {\\\"op\\\":\\\"literal\\\",\\\"value\\\":\\\"2021-04-29\\\",\\\"valueType\\\":\\\"date\\\"}" + - " ]" + - "}\"}") - .header(new Header("Content-Type", "application/json")) - .post( - "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/query", - "name", - "default", - "table1") - .then() - .statusCode(200) - .extract() - .body() - .asString() - .split("\n"); + .when() + .filter(deltaFilter) + .body("{\"jsonPredicateHints\": \"{" + " \\\"op\\\": \\\"equal\\\"," + + " \\\"children\\\": [" + + " {\\\"op\\\": \\\"column\\\", \\\"name\\\":\\\"date\\\", \\\"valueType\\\":\\\"date\\\"}," + + " {\\\"op\\\":\\\"literal\\\",\\\"value\\\":\\\"2021-04-29\\\",\\\"valueType\\\":\\\"date\\\"}" + + " ]" + + "}\"}") + .header(new Header("Content-Type", "application/json")) + .post( + "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/query", + "name", + "default", + "table1") + .then() + .statusCode(200) + .extract() + .body() + .asString() + .split("\n"); assertEquals( - deltaTable1Protocol, - objectMapper.reader().readValue(responseBodyLines[0], ProtocolObject.class)); + deltaTable1Protocol, + objectMapper.reader().readValue(responseBodyLines[0], ProtocolObject.class)); assertEquals( - deltaTable1Metadata, - objectMapper.reader().readValue(responseBodyLines[1], MetadataObject.class)); + deltaTable1Metadata, + objectMapper.reader().readValue(responseBodyLines[1], MetadataObject.class)); var files = Arrays.stream(responseBodyLines) - .skip(2) - .map(line -> { - try { - return objectMapper - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .reader() - .readValue(line, FileObjectWithoutPresignedUrl.class); - } catch (IOException e) { - throw new RuntimeException(e); - } - }) - .collect(Collectors.toSet()); + .skip(2) + .map(line -> { + try { + return objectMapper + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + .reader() + .readValue(line, FileObjectWithoutPresignedUrl.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + }) + .collect(Collectors.toSet()); assertEquals(7, responseBodyLines.length); assertEquals(deltaTable1FilesWithoutPresignedUrl, files); // TOD } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java index 9e84f22ac..682a5bfa0 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java @@ -88,11 +88,7 @@ default void validateChildren(List children) throws PredicateException { // otherwise cannot throw exception in method call of lambda for (BaseOp c : children) { - try { - c.validate(); - } catch (PredicateException e) { - throw new RuntimeException(e); - } + c.validate(); } var child1 = children.get(0); diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java index dc4f3dc54..12d651549 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java @@ -57,7 +57,7 @@ public Object eval(EvalContext ctx) { public void validate() throws PredicateException { if (name == null) { - throw new IllegalArgumentException("Name must be specified: " + this); + throw new NonExistingColumnException("Name must be specified: " + this); } if (!this.isSupportedType(valueType, V1)) { throw new TypeNotSupportedException(valueType); diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java index 2d2611cd8..866efb0e8 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java @@ -37,11 +37,13 @@ public static NonLeafOp createPartitionFilter(List children, String oper return new GreaterThanOp(children); case ">=": return new GreaterThanOrEqualOp(children); + // case "<>": + // return new DifferentThanOp(children); case "isnull": return new IsNullOp(children); default: // TODO: add not supported sql exception - throw new PredicateException(); + throw new ExpressionNotSupportedException(operator); } } @@ -97,11 +99,7 @@ public void validate() throws PredicateException { @Override public Object eval(EvalContext ctx) throws PredicateException { - try { - this.validate(); - } catch (PredicateException e) { - throw new RuntimeException(e); - } + this.validate(); return EvalHelper.equal(children, ctx); } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/TypeMismatchException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/TypeMismatchException.java index 58093d5f5..45f585bd5 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/TypeMismatchException.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/TypeMismatchException.java @@ -14,6 +14,6 @@ public TypeMismatchException(DataType lType, DataType rType) { @Override public String getMessage() { - return "Type are not matching between: " + lType.toString() + "and " + rType; + return "Type are not matching between: " + lType.toString() + " and " + rType; } } diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index 50f8441e6..adbd63290 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -90,25 +90,6 @@ void queryTableWithoutPredicate() { assertEquals(response.other().size(), 9); } - @Test - void queryTableWithJsonPredicate() { - var predicate = "{" - + " \"op\":\"equal\",\n" - + " \"children\":[\n" - + " {\"op\":\"column\",\"name\":\"date\",\"valueType\":\"date\"},\n" - + " {\"op\":\"literal\",\"value\":\"2021-08-15\",\"valueType\":\"date\"}\n" - + " ]\n" - + "}"; - - var PTable = new SharedTable( - "partitioned-delta-table", "default", "share1", deltaTable("partitioned-delta-table")); - var DTable = DeltaSharedTable.of(PTable); - var request = new ReadTableRequest.ReadTableCurrentVersion( - Optional.empty(), Optional.of(predicate), Optional.empty()); - var response = DTable.queryTable(request); - assertEquals(response.other().size(), 4); - } - @Test void queryTableWithSqlPredicates() { @@ -146,7 +127,7 @@ void queryTableWithNonPartitionSqlPredicate() { } @Test - void queryTableWithInvalidJsonPredicate() { + void queryTableWithJsonPredicates() { var predicatesAndExpectedResult = List.of( Pair.of( diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateExceptionsTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateExceptionsTest.java new file mode 100644 index 000000000..98fa38bbd --- /dev/null +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateExceptionsTest.java @@ -0,0 +1,81 @@ +package io.whitefox.core.types.predicates; + +import static io.whitefox.DeltaTestUtils.deltaTableUri; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import io.delta.standalone.DeltaLog; +import io.delta.standalone.actions.AddFile; +import io.whitefox.core.PredicateUtils; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Test; + +public class PredicateExceptionsTest { + + DeltaLog log = DeltaLog.forTable( + new Configuration(), deltaTableUri("partitioned-delta-table-with-multiple-columns")); + AddFile file = log.snapshot().getAllFiles().get(0); + + @Test + void testTypeNotSupportedExceptionGettingThrown() throws PredicateParsingException { + + EvalContext context = PredicateUtils.createEvalContext(file); + var predicate = "{" + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"dating\",\"valueType\":\"float\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-09-12\",\"valueType\":\"float\"}\n" + + " ]\n" + + "}"; + + var parsed = PredicateUtils.parseJsonPredicate(predicate); + assertThrows(TypeNotSupportedException.class, () -> parsed.evalExpectBoolean(context)); + } + + @Test + void testNonExistingColumnExceptionGettingThrown() throws PredicateParsingException { + + EvalContext context = PredicateUtils.createEvalContext(file); + var predicate = "{" + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"notPresent\",\"valueType\": \"date\"},\n" + + " {\"op\":\"literal\",\"value\":\"2021-09-12\",\"valueType\":\"date\"}\n" + + " ]\n" + + "}"; + + var parsed = PredicateUtils.parseJsonPredicate(predicate); + assertThrows(NonExistingColumnException.class, () -> parsed.evalExpectBoolean(context)); + } + + @Test + void testTypeMismatchExceptionGettingThrown() throws PredicateParsingException { + + EvalContext context = PredicateUtils.createEvalContext(file); + var predicate = "{" + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + + " {\"op\":\"literal\",\"value\":\"abcd\",\"valueType\":\"string\"}\n" + + " ]\n" + + "}"; + + var parsed = PredicateUtils.parseJsonPredicate(predicate); + assertThrows(TypeMismatchException.class, () -> parsed.evalExpectBoolean(context)); + } + + @Test + void testTypeValidationExceptionGettingThrown() throws PredicateParsingException { + + EvalContext context = PredicateUtils.createEvalContext(file); + var predicate = "{" + + " \"op\":\"equal\",\n" + + " \"children\":[\n" + + " {\"op\":\"column\",\"name\":\"id\",\"valueType\":\"int\"},\n" + + " {\"op\":\"literal\",\"value\":\"abcd\",\"valueType\":\"int\"}\n" + + " ]\n" + + "}"; + + var parsed = PredicateUtils.parseJsonPredicate(predicate); + assertThrows(TypeValidationException.class, () -> parsed.evalExpectBoolean(context)); + } +} diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java index 88025f463..f70ea8730 100644 --- a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateParsingTest.java @@ -30,6 +30,29 @@ void testParsingOfJsonEqual() throws PredicateException { assert (((EqualOp) op).children.size() == 2); } + @Test + void testParsingOfInvalidSql() { + var meta = new Metadata( + "id", + Optional.empty(), + Optional.empty(), + Metadata.Format.PARQUET, + new TableSchema( + new StructType(List.of(new StructField("date", DateType.DATE, true, Map.of())))), + List.of("date", "age"), + Map.of(), + Optional.empty(), + Optional.empty(), + Optional.empty()); + + var ctx = new EvalContext(Map.of("date", "date"), Map.of()); + var predicate = "date LIKE '2021-09-09'"; + + assertThrows( + ExpressionNotSupportedException.class, + () -> PredicateUtils.parseSqlPredicate(predicate, ctx, meta)); + } + @Test void testParsingOfSqlEqual() throws PredicateException { var ctx = new EvalContext(Map.of("date", "date"), Map.of()); From f5a076aed1dabb62765093bc8d2b9834fd69fb8c Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 19 Jan 2024 02:30:33 +0100 Subject: [PATCH 21/30] disable some tests on windows --- .../whitefox/core/services/DeltaSharedTableTest.java | 11 +++++------ .../io/whitefox/core/types/PredicateUtilsTest.java | 3 +++ .../types/predicates/PredicateExceptionsTest.java | 3 +++ 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index adbd63290..40258fbd7 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -13,7 +13,6 @@ import java.time.format.DateTimeParseException; import java.util.List; import java.util.Optional; -import java.util.concurrent.ExecutionException; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.condition.DisabledOnOs; @@ -23,7 +22,7 @@ public class DeltaSharedTableTest { @Test - void getTableVersion() throws ExecutionException, InterruptedException { + void getTableVersion() { var PTable = new SharedTable("delta-table", "default", "share1", deltaTable("delta-table")); var DTable = DeltaSharedTable.of(PTable); var version = DTable.getTableVersion(Optional.empty()); @@ -46,7 +45,7 @@ void getUnknownTableMetadata() { } @Test - void getTableVersionNonExistingTable() throws ExecutionException, InterruptedException { + void getTableVersionNonExistingTable() { var PTable = new SharedTable("delta-table", "default", "share1", deltaTable("delta-table-not-exists")); var exception = assertThrows(IllegalArgumentException.class, () -> DeltaSharedTable.of(PTable)); @@ -54,7 +53,7 @@ void getTableVersionNonExistingTable() throws ExecutionException, InterruptedExc } @Test - void getTableVersionWithTimestamp() throws ExecutionException, InterruptedException { + void getTableVersionWithTimestamp() { var PTable = new SharedTable("delta-table", "default", "share1", deltaTable("delta-table")); var DTable = DeltaSharedTable.of(PTable); var version = DTable.getTableVersion(TestDateUtils.parseTimestamp("2023-09-30T10:15:30+01:00")); @@ -62,7 +61,7 @@ void getTableVersionWithTimestamp() throws ExecutionException, InterruptedExcept } @Test - void getTableVersionWithFutureTimestamp() throws ExecutionException, InterruptedException { + void getTableVersionWithFutureTimestamp() { var PTable = new SharedTable("delta-table", "default", "share1", deltaTable("delta-table")); var DTable = DeltaSharedTable.of(PTable); var version = DTable.getTableVersion(TestDateUtils.parseTimestamp("2024-10-20T10:15:30+01:00")); @@ -70,7 +69,7 @@ void getTableVersionWithFutureTimestamp() throws ExecutionException, Interrupted } @Test - void getTableVersionWithMalformedTimestamp() throws ExecutionException, InterruptedException { + void getTableVersionWithMalformedTimestamp() { var PTable = new SharedTable("delta-table", "default", "share1", deltaTable("delta-table")); var DTable = DeltaSharedTable.of(PTable); assertThrows( diff --git a/server/core/src/test/java/io/whitefox/core/types/PredicateUtilsTest.java b/server/core/src/test/java/io/whitefox/core/types/PredicateUtilsTest.java index 8c21e9d8d..9e4b63c70 100644 --- a/server/core/src/test/java/io/whitefox/core/types/PredicateUtilsTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/PredicateUtilsTest.java @@ -11,10 +11,13 @@ import java.util.ArrayList; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; public class PredicateUtilsTest { @Test + @DisabledOnOs(OS.WINDOWS) void testCreateEvalContext() throws PredicateParsingException { var PTable = new SharedTable( "partitioned-delta-table-with-multiple-columns", diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateExceptionsTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateExceptionsTest.java index 98fa38bbd..e78a77850 100644 --- a/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateExceptionsTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/PredicateExceptionsTest.java @@ -8,7 +8,10 @@ import io.whitefox.core.PredicateUtils; import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledOnOs; +import org.junit.jupiter.api.condition.OS; +@DisabledOnOs(OS.WINDOWS) public class PredicateExceptionsTest { DeltaLog log = DeltaLog.forTable( From 8f3c7e5e911928957574a34e6c3bf7daad57a8e5 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 19 Jan 2024 03:17:21 +0100 Subject: [PATCH 22/30] more tests --- .../core/types/predicates/NonLeafOp.java | 34 +++++++++++-- .../core/services/DeltaShareServiceTest.java | 49 +++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java index 866efb0e8..1ab637f1b 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java @@ -17,7 +17,8 @@ @JsonSubTypes.Type(value = LessThanOp.class, name = "lessThan"), @JsonSubTypes.Type(value = LessThanOrEqualOp.class, name = "lessThanOrEqual"), @JsonSubTypes.Type(value = GreaterThanOp.class, name = "greaterThan"), - @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqual") + @JsonSubTypes.Type(value = GreaterThanOrEqualOp.class, name = "greaterThanOrEqual"), + @JsonSubTypes.Type(value = DifferentThanOp.class, name = "differentThan") }) public abstract class NonLeafOp implements BaseOp { @@ -37,8 +38,8 @@ public static NonLeafOp createPartitionFilter(List children, String oper return new GreaterThanOp(children); case ">=": return new GreaterThanOrEqualOp(children); - // case "<>": - // return new DifferentThanOp(children); + case "<>": + return new DifferentThanOp(children); case "isnull": return new IsNullOp(children); default: @@ -104,6 +105,33 @@ public Object eval(EvalContext ctx) throws PredicateException { } } +// not used in JsonPredicates, only for SQL +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "differentThan") +class DifferentThanOp extends NonLeafOp implements BinaryOp { + @JsonProperty("children") + List children; + + public DifferentThanOp() { + super(); + } + + public DifferentThanOp(List children) { + this.children = children; + } + + @Override + public void validate() throws PredicateException { + validateChildren( + List.copyOf(children).stream().map(c -> (BaseOp) c).collect(Collectors.toList())); + } + + @Override + public Object eval(EvalContext ctx) throws PredicateException { + this.validate(); + return EvalHelper.equal(children, ctx); + } +} + @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "lessThan") class LessThanOp extends NonLeafOp implements BinaryOp { @JsonProperty("children") diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaShareServiceTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaShareServiceTest.java index 676d45a9b..b62a411f6 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaShareServiceTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaShareServiceTest.java @@ -5,6 +5,8 @@ import io.whitefox.core.Schema; import io.whitefox.core.Share; import io.whitefox.core.SharedTable; +import io.whitefox.core.*; +import io.whitefox.core.services.exceptions.TableNotFound; import io.whitefox.persistence.StorageManager; import io.whitefox.persistence.memory.InMemoryStorageManager; import java.util.Collections; @@ -228,4 +230,51 @@ public void tableMetadataNotFound() { deltaSharesService.getTableMetadata("name", "default", "tableNotFound", Optional.empty()); Assertions.assertTrue(resultTable.isEmpty()); } + + @Test + public void queryExistingTable() { + var shares = List.of(createShare( + "name", + "key", + Map.of( + "default", + new Schema( + "default", + List.of(new SharedTable( + "partitioned-delta-table", + "default", + "name", + DeltaTestUtils.deltaTable("partitioned-delta-table"))), + "name")))); + StorageManager storageManager = new InMemoryStorageManager(shares); + DeltaSharesService deltaSharesService = + new DeltaSharesServiceImpl(storageManager, 100, tableLoaderFactory, fileSignerFactory); + var resultTable = deltaSharesService.queryTable( + "name", + "default", + "partitioned-delta-table", + new ReadTableRequest.ReadTableCurrentVersion( + Optional.empty(), Optional.empty(), Optional.empty())); + Assertions.assertEquals(9, resultTable.files().size()); + } + + @Test + public void queryNonExistingTable() { + var shares = List.of(createShare( + "name", + "key", + Map.of( + "default", + new Schema( + "default", + List.of(new SharedTable( + "table1", "default", "name", DeltaTestUtils.deltaTable("location1"))), + "name")))); + StorageManager storageManager = new InMemoryStorageManager(shares); + DeltaSharesService deltaSharesService = + new DeltaSharesServiceImpl(storageManager, 100, loader, fileSignerFactory); + Assertions.assertThrows( + TableNotFound.class, + () -> deltaSharesService.queryTable("name", "default", "tableNotFound", null)); + } } From 8c79a2518660e951f47a9fb92190026996fe8358 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 19 Jan 2024 03:49:59 +0100 Subject: [PATCH 23/30] more tests and coverage printer --- .../core/types/predicates/BaseOp.java | 6 +--- .../core/types/predicates/EvalHelperTest.java | 36 +++++++++++++++++++ 2 files changed, 37 insertions(+), 5 deletions(-) create mode 100644 server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java index 682a5bfa0..c95bb0fb9 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/BaseOp.java @@ -112,11 +112,7 @@ default void validateChildren(List children) throws PredicateException { } for (BaseOp c : children) { - try { - c.validate(); - } catch (PredicateException e) { - throw new RuntimeException(e); - } + c.validate(); } } } diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java new file mode 100644 index 000000000..d693626af --- /dev/null +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java @@ -0,0 +1,36 @@ +package io.whitefox.core.types.predicates; + +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.whitefox.core.types.DateType; +import io.whitefox.core.types.IntegerType; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.Test; + +public class EvalHelperTest { + + @Test + void testLessThan() throws PredicateException { + var evalContext1 = new EvalContext(Map.of("date", "2020-10-10"), Map.of()); + var children1 = + List.of(new ColumnOp("date", DateType.DATE), new LiteralOp("2020-10-11", DateType.DATE)); + assertTrue(EvalHelper.lessThan(children1, evalContext1)); + var evalContext2 = new EvalContext(Map.of("integer", "19"), Map.of()); + var children2 = List.of( + new ColumnOp("integer", IntegerType.INTEGER), new LiteralOp("20", IntegerType.INTEGER)); + assertTrue(EvalHelper.lessThan(children2, evalContext2)); + } + + @Test + void testEqual() throws PredicateException { + var evalContext1 = new EvalContext(Map.of("date", "2020-10-11"), Map.of()); + var children1 = + List.of(new ColumnOp("date", DateType.DATE), new LiteralOp("2020-10-11", DateType.DATE)); + assertTrue(EvalHelper.equal(children1, evalContext1)); + var evalContext2 = new EvalContext(Map.of("integer", "20"), Map.of()); + var children2 = List.of( + new ColumnOp("integer", IntegerType.INTEGER), new LiteralOp("20", IntegerType.INTEGER)); + assertTrue(EvalHelper.equal(children2, evalContext2)); + } +} From 79179fc9ceacd55c248eec4b9c123e47cdadb969 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 19 Jan 2024 16:00:05 +0100 Subject: [PATCH 24/30] more tests + spotless --- .../java/io/whitefox/core/ColumnRange.java | 2 + .../core/types/predicates/EvalHelper.java | 6 +- .../core/types/predicates/NonLeafOp.java | 2 +- .../PredicateColumnEvaluationException.java | 19 +++ .../core/services/DeltaSharedTableTest.java | 1 + .../core/types/predicates/EvalHelperTest.java | 132 +++++++++++++++++- 6 files changed, 153 insertions(+), 9 deletions(-) create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/PredicateColumnEvaluationException.java diff --git a/server/core/src/main/java/io/whitefox/core/ColumnRange.java b/server/core/src/main/java/io/whitefox/core/ColumnRange.java index 4ea3c2361..9104d9513 100644 --- a/server/core/src/main/java/io/whitefox/core/ColumnRange.java +++ b/server/core/src/main/java/io/whitefox/core/ColumnRange.java @@ -1,5 +1,6 @@ package io.whitefox.core; +import io.whitefox.annotations.SkipCoverageGenerated; import io.whitefox.core.types.*; import java.sql.Date; import java.sql.Timestamp; @@ -82,6 +83,7 @@ private Boolean typedLessThan(String point) { } } + @SkipCoverageGenerated private Boolean typedGreaterThan(String point) { if (valueType instanceof IntegerType) { var c = Integer.compare(Integer.parseInt(point), Integer.parseInt(maxVal)); diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index a98a90a54..875fa8f13 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -67,6 +67,7 @@ else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { var leftVal = typesAndValues.getLeft().getRight(); var rightVal = typesAndValues.getRight().getRight(); + // we fear no exception here since it is validated before if (BooleanType.BOOLEAN.equals(leftType)) { return Boolean.valueOf(leftVal) == Boolean.valueOf(rightVal); } else if (IntegerType.INTEGER.equals(leftType)) { @@ -78,7 +79,7 @@ else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { } else if (DateType.DATE.equals(leftType)) { return Date.valueOf(leftVal).equals(Date.valueOf(rightVal)); } else throw new TypeNotSupportedException(leftType); - } else throw new PredicateException(); + } else throw new PredicateColumnEvaluationException(ctx); } static Boolean lessThan(List children, EvalContext ctx) throws PredicateException { @@ -106,7 +107,7 @@ else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { } else if (DateType.DATE.equals(leftType)) { return Date.valueOf(leftVal).before(Date.valueOf(rightVal)); } else throw new TypeNotSupportedException(leftType); - } else throw new PredicateException(); + } else throw new PredicateColumnEvaluationException(ctx); } // Validates that the specified value is in the correct format. @@ -126,7 +127,6 @@ public static void validateValue(String value, DataType valueType) Float.parseFloat(value); } else if (DoubleType.DOUBLE.equals(valueType)) { Double.parseDouble(value); - // TODO check for non deprecated } else if (TimestampType.TIMESTAMP.equals(valueType)) { Timestamp.valueOf(value); } else if (StringType.STRING.equals(valueType)) { diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java index 1ab637f1b..69b20957f 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/NonLeafOp.java @@ -128,7 +128,7 @@ public void validate() throws PredicateException { @Override public Object eval(EvalContext ctx) throws PredicateException { this.validate(); - return EvalHelper.equal(children, ctx); + return !EvalHelper.equal(children, ctx); } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateColumnEvaluationException.java b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateColumnEvaluationException.java new file mode 100644 index 000000000..ceaedd1f0 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/PredicateColumnEvaluationException.java @@ -0,0 +1,19 @@ +package io.whitefox.core.types.predicates; + +public class PredicateColumnEvaluationException extends PredicateException { + + private final EvalContext ctx; + + public PredicateColumnEvaluationException(EvalContext ctx) { + this.ctx = ctx; + } + + @Override + public String getMessage() { + return "Column from your query does not exist in either partition columns or regular columns of the table." + + "Partition columns: " + + ctx.getPartitionValues().keySet().stream().reduce((s1, s2) -> s1 + "|" + s2) + "\n" + + "Regular columns: " + + ctx.getStatsValues().keySet().stream().reduce((s1, s2) -> s1 + "|" + s2) + "\n"; + } +} diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index 40258fbd7..bea163cad 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -98,6 +98,7 @@ void queryTableWithSqlPredicates() { Pair.of(List.of("date > '2021-08-04'"), 9), Pair.of(List.of("date is NULL"), 0), Pair.of(List.of("date >= '2021-08-15'"), 4), + Pair.of(List.of("date <> '2021-08-15'"), 5), Pair.of(List.of("date <= '2021-08-15'"), 9)); var PTable = new SharedTable( diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java index d693626af..cd8d24a50 100644 --- a/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java @@ -1,17 +1,17 @@ package io.whitefox.core.types.predicates; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; -import io.whitefox.core.types.DateType; -import io.whitefox.core.types.IntegerType; +import io.whitefox.core.types.*; import java.util.List; import java.util.Map; +import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; public class EvalHelperTest { @Test - void testLessThan() throws PredicateException { + void testLessThanOnPartitionColumns() throws PredicateException { var evalContext1 = new EvalContext(Map.of("date", "2020-10-10"), Map.of()); var children1 = List.of(new ColumnOp("date", DateType.DATE), new LiteralOp("2020-10-11", DateType.DATE)); @@ -20,10 +20,19 @@ void testLessThan() throws PredicateException { var children2 = List.of( new ColumnOp("integer", IntegerType.INTEGER), new LiteralOp("20", IntegerType.INTEGER)); assertTrue(EvalHelper.lessThan(children2, evalContext2)); + var evalContext3 = new EvalContext(Map.of("long", "20"), Map.of()); + var children3 = + List.of(new ColumnOp("long", LongType.LONG), new LiteralOp("21", LongType.LONG)); + assertTrue(EvalHelper.lessThan(children3, evalContext3)); + var evalContext4 = new EvalContext(Map.of("float", "2.99"), Map.of()); + var children4 = + List.of(new ColumnOp("float", FloatType.FLOAT), new LiteralOp("2.98", FloatType.FLOAT)); + assertThrows( + TypeNotSupportedException.class, () -> EvalHelper.lessThan(children4, evalContext4)); } @Test - void testEqual() throws PredicateException { + void testEqualOnPartitionColumns() throws PredicateException { var evalContext1 = new EvalContext(Map.of("date", "2020-10-11"), Map.of()); var children1 = List.of(new ColumnOp("date", DateType.DATE), new LiteralOp("2020-10-11", DateType.DATE)); @@ -32,5 +41,118 @@ void testEqual() throws PredicateException { var children2 = List.of( new ColumnOp("integer", IntegerType.INTEGER), new LiteralOp("20", IntegerType.INTEGER)); assertTrue(EvalHelper.equal(children2, evalContext2)); + var evalContext3 = new EvalContext(Map.of("long", "20"), Map.of()); + var children3 = + List.of(new ColumnOp("long", LongType.LONG), new LiteralOp("20", LongType.LONG)); + assertTrue(EvalHelper.equal(children3, evalContext3)); + var evalContext4 = new EvalContext(Map.of("boolean", "true"), Map.of()); + var children4 = List.of( + new ColumnOp("boolean", BooleanType.BOOLEAN), new LiteralOp("true", BooleanType.BOOLEAN)); + assertTrue(EvalHelper.equal(children4, evalContext4)); + var evalContext5 = new EvalContext(Map.of("float", "2.99"), Map.of()); + var children5 = + List.of(new ColumnOp("float", FloatType.FLOAT), new LiteralOp("2.99", FloatType.FLOAT)); + assertThrows(TypeNotSupportedException.class, () -> EvalHelper.equal(children5, evalContext5)); + } + + @Test + void testEqualOnRegularColumns() throws PredicateException { + var evalContext1 = new EvalContext( + Map.of("date", "2020-10-11"), Map.of("id", Pair.of("2020-11-10", "2020-11-12"))); + var children1 = + List.of(new ColumnOp("id", DateType.DATE), new LiteralOp("2020-11-11", DateType.DATE)); + assertTrue(EvalHelper.equal(children1, evalContext1)); + var evalContext2 = new EvalContext(Map.of("integer", "20"), Map.of("id", Pair.of("20", "29"))); + var children2 = + List.of(new ColumnOp("id", IntegerType.INTEGER), new LiteralOp("20", IntegerType.INTEGER)); + assertTrue(EvalHelper.equal(children2, evalContext2)); + var evalContext3 = new EvalContext(Map.of("long", "20"), Map.of("id", Pair.of("20", "29"))); + var children3 = List.of(new ColumnOp("id", LongType.LONG), new LiteralOp("21", LongType.LONG)); + assertTrue(EvalHelper.equal(children3, evalContext3)); + var evalContext4 = new EvalContext(Map.of("long", "20"), Map.of("id", Pair.of("2.99", "3.01"))); + var children4 = + List.of(new ColumnOp("id", FloatType.FLOAT), new LiteralOp("3.0", FloatType.FLOAT)); + assertTrue(EvalHelper.equal(children4, evalContext4)); + var evalContext5 = new EvalContext(Map.of("long", "20"), Map.of("id", Pair.of("aaaa", "cccc"))); + var children5 = + List.of(new ColumnOp("id", StringType.STRING), new LiteralOp("aabb", StringType.STRING)); + assertTrue(EvalHelper.equal(children5, evalContext5)); + var evalContext6 = new EvalContext(Map.of("long", "20"), Map.of("id", Pair.of("2.99", "3.01"))); + var children6 = + List.of(new ColumnOp("id", DoubleType.DOUBLE), new LiteralOp("3.0", DoubleType.DOUBLE)); + assertTrue(EvalHelper.equal(children6, evalContext6)); + var evalContext7 = new EvalContext(Map.of("long", "20"), Map.of("id", Pair.of("true", "true"))); + var children7 = List.of( + new ColumnOp("id", BooleanType.BOOLEAN), new LiteralOp("true", BooleanType.BOOLEAN)); + assertTrue(EvalHelper.equal(children7, evalContext7)); + var evalContext8 = new EvalContext( + Map.of("long", "20"), + Map.of("id", Pair.of("2022-08-10 06:02:03.000000", "2022-12-10 06:02:03.000000"))); + var children8 = List.of( + new ColumnOp("id", TimestampType.TIMESTAMP), + new LiteralOp("2022-10-10 06:02:03.000000", TimestampType.TIMESTAMP)); + assertTrue(EvalHelper.equal(children8, evalContext8)); + } + + @Test + void testLessThanOnRegularColumns() throws PredicateException { + var evalContext1 = new EvalContext( + Map.of("date", "2020-10-11"), Map.of("id", Pair.of("2020-11-10", "2020-11-12"))); + var children1 = + List.of(new ColumnOp("id", DateType.DATE), new LiteralOp("2020-12-12", DateType.DATE)); + assertTrue(EvalHelper.lessThan(children1, evalContext1)); + var evalContext2 = new EvalContext(Map.of("integer", "20"), Map.of("id", Pair.of("20", "29"))); + var children2 = + List.of(new ColumnOp("id", IntegerType.INTEGER), new LiteralOp("30", IntegerType.INTEGER)); + assertTrue(EvalHelper.lessThan(children2, evalContext2)); + var evalContext3 = new EvalContext(Map.of("long", "20"), Map.of("id", Pair.of("20", "29"))); + var children3 = List.of(new ColumnOp("id", LongType.LONG), new LiteralOp("30", LongType.LONG)); + assertTrue(EvalHelper.lessThan(children3, evalContext3)); + var evalContext4 = new EvalContext(Map.of("long", "20"), Map.of("id", Pair.of("2.99", "3.01"))); + var children4 = + List.of(new ColumnOp("id", FloatType.FLOAT), new LiteralOp("3.02", FloatType.FLOAT)); + assertTrue(EvalHelper.lessThan(children4, evalContext4)); + var evalContext5 = new EvalContext(Map.of("long", "20"), Map.of("id", Pair.of("2.99", "3.01"))); + var children5 = + List.of(new ColumnOp("id", DoubleType.DOUBLE), new LiteralOp("3.02", DoubleType.DOUBLE)); + assertTrue(EvalHelper.lessThan(children5, evalContext5)); + var evalContext6 = new EvalContext(Map.of("long", "20"), Map.of("id", Pair.of("2.99", "3.01"))); + var children6 = + List.of(new ColumnOp("id", DoubleType.DOUBLE), new LiteralOp("3.02", DoubleType.DOUBLE)); + assertTrue(EvalHelper.lessThan(children6, evalContext6)); + var evalContext7 = new EvalContext(Map.of("long", "20"), Map.of("id", Pair.of("aaaa", "cccc"))); + var children7 = + List.of(new ColumnOp("id", StringType.STRING), new LiteralOp("dddd", StringType.STRING)); + assertTrue(EvalHelper.lessThan(children7, evalContext7)); + var evalContext8 = new EvalContext( + Map.of("long", "20"), + Map.of("id", Pair.of("2022-08-10 06:02:03.000000", "2022-12-10 06:02:03.000000"))); + var children8 = List.of( + new ColumnOp("id", TimestampType.TIMESTAMP), + new LiteralOp("2022-12-11 06:02:03.000000", TimestampType.TIMESTAMP)); + assertTrue(EvalHelper.lessThan(children8, evalContext8)); + } + + @Test + void testValidateAndGetRange() throws PredicateException { + var evalContext1 = + new EvalContext(Map.of("date", "2020-10-11"), Map.of("id", Pair.of("20", "29"))); + var children1 = List.of( + new ColumnOp("notId", IntegerType.INTEGER), new LiteralOp("30", IntegerType.INTEGER)); + assertThrows( + NonExistingColumnException.class, () -> EvalHelper.lessThan(children1, evalContext1)); + var children2 = + List.of(new ColumnOp("id", IntegerType.INTEGER), new LiteralOp("30", IntegerType.INTEGER)); + assertTrue(EvalHelper.lessThan(children2, evalContext1)); + assertFalse(EvalHelper.equal(children2, evalContext1)); + var reversedChildren = + List.of(new LiteralOp("30", IntegerType.INTEGER), new ColumnOp("id", IntegerType.INTEGER)); + assertFalse(EvalHelper.equal(reversedChildren, evalContext1)); + var evalContext2 = + new EvalContext(Map.of("date", "2020-10-11"), Map.of("id", Pair.of("20", "29"))); + var children3 = + List.of(new ColumnOp("notId", IntegerType.INTEGER), new LiteralOp("30", DateType.DATE)); + assertThrows(TypeMismatchException.class, () -> EvalHelper.lessThan(children3, evalContext2)); + assertThrows(TypeMismatchException.class, () -> EvalHelper.equal(children3, evalContext2)); } } From 9322aa00e88a405a0c1b6c1f0ee622802009c9b7 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 19 Jan 2024 16:04:33 +0100 Subject: [PATCH 25/30] disabled test on windows --- .../java/io/whitefox/core/services/DeltaShareServiceTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaShareServiceTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaShareServiceTest.java index b62a411f6..fa0838dc6 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaShareServiceTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaShareServiceTest.java @@ -232,6 +232,7 @@ public void tableMetadataNotFound() { } @Test + @DisabledOnOs(OS.WINDOWS) public void queryExistingTable() { var shares = List.of(createShare( "name", From 9807fd1eace5fdf0aec928ea2683b2332fcff6d4 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Mon, 29 Jan 2024 23:02:21 +0100 Subject: [PATCH 26/30] primitive type names simplified --- .../whitefox.java-conventions.gradle.kts | 2 +- .../server/DeltaSharesApiImplTest.java | 2 +- .../core/types/BasePrimitiveType.java | 2 +- .../io/whitefox/core/types/IntegerType.java | 2 +- .../predicates/BasePrimitiveTypeNames.java | 17 ------- .../predicates/DataTypeDeserializer.java | 44 +++++++++++-------- .../io/whitefox/core/types/MapTypeTest.java | 2 +- .../core/types/predicates/EvalHelperTest.java | 8 ++-- 8 files changed, 34 insertions(+), 45 deletions(-) delete mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java diff --git a/buildSrc/src/main/kotlin/whitefox.java-conventions.gradle.kts b/buildSrc/src/main/kotlin/whitefox.java-conventions.gradle.kts index 700052872..869d2e156 100644 --- a/buildSrc/src/main/kotlin/whitefox.java-conventions.gradle.kts +++ b/buildSrc/src/main/kotlin/whitefox.java-conventions.gradle.kts @@ -34,7 +34,7 @@ tasks.withType().configureEach { tasks.register("devCheck") { dependsOn(tasks.spotlessApply) - finalizedBy(tasks.test) + finalizedBy(tasks.check) description = "Useful command when iterating locally to apply spotless formatting then running all the checks" } diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java index e24bf0b20..0b268b957 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java @@ -414,7 +414,7 @@ public void queryTableCurrentVersionWithPredicates() throws IOException { }) .collect(Collectors.toSet()); assertEquals(7, responseBodyLines.length); - assertEquals(deltaTable1FilesWithoutPresignedUrl, files); // TOD + assertEquals(deltaTable1FilesWithoutPresignedUrl, files); } @DisabledOnOs(OS.WINDOWS) diff --git a/server/core/src/main/java/io/whitefox/core/types/BasePrimitiveType.java b/server/core/src/main/java/io/whitefox/core/types/BasePrimitiveType.java index 1de9ca463..0f6bb5ee3 100644 --- a/server/core/src/main/java/io/whitefox/core/types/BasePrimitiveType.java +++ b/server/core/src/main/java/io/whitefox/core/types/BasePrimitiveType.java @@ -38,7 +38,7 @@ public static List getAllPrimitiveTypes() { put("boolean", BooleanType.BOOLEAN); put("byte", ByteType.BYTE); put("short", ShortType.SHORT); - put("integer", IntegerType.INTEGER); + put("int", IntegerType.INTEGER); put("long", LongType.LONG); put("float", FloatType.FLOAT); put("double", DoubleType.DOUBLE); diff --git a/server/core/src/main/java/io/whitefox/core/types/IntegerType.java b/server/core/src/main/java/io/whitefox/core/types/IntegerType.java index 546426a86..d8b1040f7 100644 --- a/server/core/src/main/java/io/whitefox/core/types/IntegerType.java +++ b/server/core/src/main/java/io/whitefox/core/types/IntegerType.java @@ -7,6 +7,6 @@ public class IntegerType extends BasePrimitiveType { public static final IntegerType INTEGER = new IntegerType(); private IntegerType() { - super("integer"); + super("int"); } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java b/server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java deleted file mode 100644 index 446e0eef1..000000000 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/BasePrimitiveTypeNames.java +++ /dev/null @@ -1,17 +0,0 @@ -package io.whitefox.core.types.predicates; - -public enum BasePrimitiveTypeNames { - DATE("date"), - INT("int"), - FLOAT("float"), - DOUBLE("double"), - TIMESTAMP("timestamp"), - LONG("long"), - STRING("string"); - - public final String value; - - BasePrimitiveTypeNames(String value) { - this.value = value; - } -} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java b/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java index c3af594d5..2b81a8d5a 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/DataTypeDeserializer.java @@ -1,5 +1,13 @@ package io.whitefox.core.types.predicates; +import static io.whitefox.core.types.DateType.DATE; +import static io.whitefox.core.types.DoubleType.DOUBLE; +import static io.whitefox.core.types.FloatType.FLOAT; +import static io.whitefox.core.types.IntegerType.INTEGER; +import static io.whitefox.core.types.LongType.LONG; +import static io.whitefox.core.types.StringType.STRING; +import static io.whitefox.core.types.TimestampType.TIMESTAMP; + import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.databind.DeserializationContext; @@ -23,25 +31,23 @@ public DataTypeDeserializer(Class vc) { public DataType deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { JsonNode node = jsonParser.getCodec().readTree(jsonParser); - String valueType = node.asText().toUpperCase(); - - switch (BasePrimitiveTypeNames.valueOf(valueType)) { - case DATE: - return DateType.DATE; - case INT: - return IntegerType.INTEGER; - case DOUBLE: - return DoubleType.DOUBLE; - case FLOAT: - return FloatType.FLOAT; - case STRING: - return StringType.STRING; - case TIMESTAMP: - return TimestampType.TIMESTAMP; - case LONG: - return LongType.LONG; - default: - throw new JsonParseException("Unknown type passed inside a json predicate: " + valueType); + String valueType = node.asText(); + DataType primitive = BasePrimitiveType.createPrimitive(valueType); + if (DATE.equals(primitive)) { + return DATE; + } else if (INTEGER.equals(primitive)) { + return INTEGER; + } else if (DOUBLE.equals(primitive)) { + return DOUBLE; + } else if (FLOAT.equals(primitive)) { + return FLOAT; + } else if (STRING.equals(primitive)) { + return STRING; + } else if (TIMESTAMP.equals(primitive)) { + return TIMESTAMP; + } else if (LONG.equals(primitive)) { + return LONG; } + throw new JsonParseException("Unknown type passed inside a json predicate: " + valueType); } } diff --git a/server/core/src/test/java/io/whitefox/core/types/MapTypeTest.java b/server/core/src/test/java/io/whitefox/core/types/MapTypeTest.java index 7863da510..223273c81 100644 --- a/server/core/src/test/java/io/whitefox/core/types/MapTypeTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/MapTypeTest.java @@ -83,7 +83,7 @@ public void testToJson() { MapType mapType = new MapType(keyType, valueType, valueContainsNull); assertEquals( - "{\"type\": \"map\",\"keyType\": \"string\",\"valueType\": \"integer\",\"valueContainsNull\": true}", + "{\"type\": \"map\",\"keyType\": \"string\",\"valueType\": \"int\",\"valueContainsNull\": true}", mapType.toJson()); } } diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java index cd8d24a50..ce0a07fb7 100644 --- a/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java @@ -16,9 +16,9 @@ void testLessThanOnPartitionColumns() throws PredicateException { var children1 = List.of(new ColumnOp("date", DateType.DATE), new LiteralOp("2020-10-11", DateType.DATE)); assertTrue(EvalHelper.lessThan(children1, evalContext1)); - var evalContext2 = new EvalContext(Map.of("integer", "19"), Map.of()); + var evalContext2 = new EvalContext(Map.of("integerCol", "19"), Map.of()); var children2 = List.of( - new ColumnOp("integer", IntegerType.INTEGER), new LiteralOp("20", IntegerType.INTEGER)); + new ColumnOp("integerCol", IntegerType.INTEGER), new LiteralOp("20", IntegerType.INTEGER)); assertTrue(EvalHelper.lessThan(children2, evalContext2)); var evalContext3 = new EvalContext(Map.of("long", "20"), Map.of()); var children3 = @@ -37,9 +37,9 @@ void testEqualOnPartitionColumns() throws PredicateException { var children1 = List.of(new ColumnOp("date", DateType.DATE), new LiteralOp("2020-10-11", DateType.DATE)); assertTrue(EvalHelper.equal(children1, evalContext1)); - var evalContext2 = new EvalContext(Map.of("integer", "20"), Map.of()); + var evalContext2 = new EvalContext(Map.of("integerCol", "20"), Map.of()); var children2 = List.of( - new ColumnOp("integer", IntegerType.INTEGER), new LiteralOp("20", IntegerType.INTEGER)); + new ColumnOp("integerCol", IntegerType.INTEGER), new LiteralOp("20", IntegerType.INTEGER)); assertTrue(EvalHelper.equal(children2, evalContext2)); var evalContext3 = new EvalContext(Map.of("long", "20"), Map.of()); var children3 = From fdcb95429a44f184422eb738301caabf19c85beb Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Mon, 29 Jan 2024 23:20:47 +0100 Subject: [PATCH 27/30] return types refactoring --- .../java/io/whitefox/core/ColumnRange.java | 8 ++++++ .../core/types/predicates/EvalHelper.java | 28 +++++++++---------- .../predicates/LeafEvaluationResult.java | 16 ++++------- .../predicates/PartitionEvaluationResult.java | 22 +++++++++++++++ .../predicates/RangeEvaluationResult.java | 22 +++++++++++++++ 5 files changed, 72 insertions(+), 24 deletions(-) create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/PartitionEvaluationResult.java create mode 100644 server/core/src/main/java/io/whitefox/core/types/predicates/RangeEvaluationResult.java diff --git a/server/core/src/main/java/io/whitefox/core/ColumnRange.java b/server/core/src/main/java/io/whitefox/core/ColumnRange.java index 9104d9513..3d42efd85 100644 --- a/server/core/src/main/java/io/whitefox/core/ColumnRange.java +++ b/server/core/src/main/java/io/whitefox/core/ColumnRange.java @@ -24,6 +24,14 @@ public ColumnRange(String onlyVal, DataType valueType) { this.valueType = valueType; } + public DataType getValueType() { + return valueType; + } + + public String getOnlyValue() { + return minVal; + } + private Boolean typedContains(String point) { if (valueType instanceof IntegerType) { var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point)); diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index 875fa8f13..0e9bba84a 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -1,11 +1,11 @@ package io.whitefox.core.types.predicates; +import io.whitefox.core.ColumnRange; import io.whitefox.core.types.*; import java.sql.Date; import java.sql.Timestamp; import java.util.List; import java.util.Objects; -import org.apache.commons.lang3.tuple.Pair; // Only for partition values public class EvalHelper { @@ -15,7 +15,7 @@ private static LeafEvaluationResult validateAndGetRange( var columnRange = columnChild.evalExpectColumnRange(ctx); var rightVal = literalChild.evalExpectValueAndType(ctx).getLeft(); - return LeafEvaluationResult.createFromRange(Pair.of(columnRange, rightVal)); + return LeafEvaluationResult.createFromRange(new RangeEvaluationResult(columnRange, rightVal)); } private static LeafEvaluationResult validateAndGetTypeAndValue( @@ -47,8 +47,8 @@ private static LeafEvaluationResult validateAndGetTypeAndValue( if (leftVal == null || rightVal == null) { throw new NullTypeException(leftChild, rightChild); } - return LeafEvaluationResult.createFromPartitionColumn( - Pair.of(Pair.of(leftType, leftVal), Pair.of(rightType, rightVal))); + return LeafEvaluationResult.createFromPartitionColumn(new PartitionEvaluationResult( + new ColumnRange(leftVal, leftType), new ColumnRange(rightVal, rightType))); } // Implements "equal" between two leaf operations. @@ -56,16 +56,16 @@ static Boolean equal(List children, EvalContext ctx) throws PredicateExc var leafEvaluationResult = validateAndGetTypeAndValue(children, ctx); var rangeEvaluation = leafEvaluationResult.rangeEvaluationResult.map(range -> { - var columnRange = range.getLeft(); - var value = range.getRight(); + var columnRange = range.getColumnRange(); + var value = range.getValue(); return columnRange.contains(value); }); if (rangeEvaluation.isPresent()) return rangeEvaluation.get(); else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { var typesAndValues = leafEvaluationResult.partitionEvaluationResult.get(); - var leftType = typesAndValues.getLeft().getLeft(); - var leftVal = typesAndValues.getLeft().getRight(); - var rightVal = typesAndValues.getRight().getRight(); + var leftType = typesAndValues.getPartitionValue().getValueType(); + var leftVal = typesAndValues.getPartitionValue().getOnlyValue(); + var rightVal = typesAndValues.getLiteralValue().getOnlyValue(); // we fear no exception here since it is validated before if (BooleanType.BOOLEAN.equals(leftType)) { @@ -86,17 +86,17 @@ static Boolean lessThan(List children, EvalContext ctx) throws Predicate var leafEvaluationResult = validateAndGetTypeAndValue(children, ctx); var rangeEvaluation = leafEvaluationResult.rangeEvaluationResult.map(range -> { - var columnRange = range.getLeft(); - var value = range.getRight(); + var columnRange = range.getColumnRange(); + var value = range.getValue(); return columnRange.canBeLess(value); }); if (rangeEvaluation.isPresent()) return rangeEvaluation.get(); else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { var typesAndValues = leafEvaluationResult.partitionEvaluationResult.get(); - var leftType = typesAndValues.getLeft().getLeft(); - var leftVal = typesAndValues.getLeft().getRight(); - var rightVal = typesAndValues.getRight().getRight(); + var leftType = typesAndValues.getPartitionValue().getValueType(); + var leftVal = typesAndValues.getPartitionValue().getOnlyValue(); + var rightVal = typesAndValues.getLiteralValue().getOnlyValue(); if (IntegerType.INTEGER.equals(leftType)) { return Integer.parseInt(leftVal) < Integer.parseInt(rightVal); diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafEvaluationResult.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafEvaluationResult.java index 38736a0f9..3e3ca790c 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafEvaluationResult.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafEvaluationResult.java @@ -1,29 +1,25 @@ package io.whitefox.core.types.predicates; -import io.whitefox.core.ColumnRange; -import io.whitefox.core.types.DataType; import java.util.Optional; -import org.apache.commons.lang3.tuple.Pair; public class LeafEvaluationResult { - Optional> rangeEvaluationResult; - Optional, Pair>> partitionEvaluationResult; + Optional rangeEvaluationResult; + Optional partitionEvaluationResult; public LeafEvaluationResult( - Optional> rangeEvaluationResult, - Optional, Pair>> partitionEvaluationResult) { + Optional rangeEvaluationResult, + Optional partitionEvaluationResult) { this.rangeEvaluationResult = rangeEvaluationResult; this.partitionEvaluationResult = partitionEvaluationResult; } - public static LeafEvaluationResult createFromRange( - Pair rangeEvaluationResult) { + public static LeafEvaluationResult createFromRange(RangeEvaluationResult rangeEvaluationResult) { return new LeafEvaluationResult(Optional.of(rangeEvaluationResult), Optional.empty()); } public static LeafEvaluationResult createFromPartitionColumn( - Pair, Pair> partitionEvaluationResult) { + PartitionEvaluationResult partitionEvaluationResult) { return new LeafEvaluationResult(Optional.empty(), Optional.of(partitionEvaluationResult)); } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/PartitionEvaluationResult.java b/server/core/src/main/java/io/whitefox/core/types/predicates/PartitionEvaluationResult.java new file mode 100644 index 000000000..0debdf19a --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/PartitionEvaluationResult.java @@ -0,0 +1,22 @@ +package io.whitefox.core.types.predicates; + +import io.whitefox.core.ColumnRange; + +public class PartitionEvaluationResult { + + ColumnRange partitionValue; + ColumnRange literalValue; + + public PartitionEvaluationResult(ColumnRange partitionValue, ColumnRange literalValue) { + this.partitionValue = partitionValue; + this.literalValue = literalValue; + } + + public ColumnRange getPartitionValue() { + return partitionValue; + } + + public ColumnRange getLiteralValue() { + return literalValue; + } +} diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/RangeEvaluationResult.java b/server/core/src/main/java/io/whitefox/core/types/predicates/RangeEvaluationResult.java new file mode 100644 index 000000000..3ae18b1e5 --- /dev/null +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/RangeEvaluationResult.java @@ -0,0 +1,22 @@ +package io.whitefox.core.types.predicates; + +import io.whitefox.core.ColumnRange; + +public class RangeEvaluationResult { + + ColumnRange columnRange; + String value; + + public RangeEvaluationResult(ColumnRange columnRange, String value) { + this.columnRange = columnRange; + this.value = value; + } + + public ColumnRange getColumnRange() { + return columnRange; + } + + public String getValue() { + return value; + } +} From 8f18c09e84a2e1e17e5f57fafebb4cfd72a1ae7f Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Tue, 30 Jan 2024 01:05:51 +0100 Subject: [PATCH 28/30] refactor everything to ColumnRange --- .../java/io/whitefox/core/ColumnRange.java | 24 ++-- .../core/services/DeltaSharedTable.java | 3 - .../core/types/predicates/ColumnOp.java | 3 +- .../core/types/predicates/EvalHelper.java | 118 ++++++++---------- .../core/types/predicates/LeafOp.java | 10 +- .../core/types/predicates/LiteralOp.java | 4 +- .../predicates/PartitionEvaluationResult.java | 6 +- .../core/services/DeltaShareServiceTest.java | 4 +- .../core/services/DeltaSharedTableTest.java | 4 +- .../core/types/predicates/EvalHelperTest.java | 14 ++- 10 files changed, 89 insertions(+), 101 deletions(-) diff --git a/server/core/src/main/java/io/whitefox/core/ColumnRange.java b/server/core/src/main/java/io/whitefox/core/ColumnRange.java index 3d42efd85..f6f7b2702 100644 --- a/server/core/src/main/java/io/whitefox/core/ColumnRange.java +++ b/server/core/src/main/java/io/whitefox/core/ColumnRange.java @@ -2,6 +2,7 @@ import io.whitefox.annotations.SkipCoverageGenerated; import io.whitefox.core.types.*; +import io.whitefox.core.types.predicates.TypeNotSupportedException; import java.sql.Date; import java.sql.Timestamp; @@ -28,11 +29,11 @@ public DataType getValueType() { return valueType; } - public String getOnlyValue() { + public String getSingleValue() { return minVal; } - private Boolean typedContains(String point) { + private Boolean typedContains(String point) throws TypeNotSupportedException { if (valueType instanceof IntegerType) { var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point)); var c2 = Integer.compare(Integer.parseInt(maxVal), Integer.parseInt(point)); @@ -44,7 +45,7 @@ private Boolean typedContains(String point) { } else if (valueType instanceof TimestampType) { var c1 = Timestamp.valueOf(minVal).before(Timestamp.valueOf(point)); var c2 = Timestamp.valueOf(maxVal).after(Timestamp.valueOf(point)); - return c1 && c2; + return (c1 && c2) || Timestamp.valueOf(minVal).equals(Timestamp.valueOf(point)); } else if (valueType instanceof FloatType) { var c1 = Float.compare(Float.parseFloat(minVal), Float.parseFloat(point)); var c2 = Float.compare(Float.parseFloat(maxVal), Float.parseFloat(point)); @@ -56,19 +57,19 @@ private Boolean typedContains(String point) { } else if (valueType instanceof DateType) { var c1 = Date.valueOf(minVal).before(Date.valueOf(point)); var c2 = Date.valueOf(maxVal).after(Date.valueOf(point)); - return c1 && c2; + return (c1 && c2) || Date.valueOf(minVal).equals(Date.valueOf(point)); } else if (valueType instanceof BooleanType) { var c1 = Boolean.parseBoolean(minVal) == Boolean.parseBoolean(point); var c2 = Boolean.parseBoolean(maxVal) == Boolean.parseBoolean(point); return c1 || c2; - } else { + } else if (valueType instanceof StringType) { var c1 = minVal.compareTo(point); var c2 = maxVal.compareTo(point); return (c1 <= 0 && c2 >= 0); - } + } else throw new TypeNotSupportedException(valueType); } - private Boolean typedLessThan(String point) { + private Boolean typedLessThan(String point) throws TypeNotSupportedException { if (valueType instanceof IntegerType) { var c1 = Integer.compare(Integer.parseInt(minVal), Integer.parseInt(point)); return (c1 < 0); @@ -85,12 +86,13 @@ private Boolean typedLessThan(String point) { return (c1 < 0); } else if (valueType instanceof DateType) { return Date.valueOf(minVal).before(Date.valueOf(point)); - } else { + } else if (valueType instanceof StringType) { var c = minVal.compareTo(point); return (c < 0); - } + } else throw new TypeNotSupportedException(valueType); } + // not used currently @SkipCoverageGenerated private Boolean typedGreaterThan(String point) { if (valueType instanceof IntegerType) { @@ -116,11 +118,11 @@ private Boolean typedGreaterThan(String point) { } } - public Boolean contains(String point) { + public Boolean contains(String point) throws TypeNotSupportedException { return typedContains(point); } - public Boolean canBeLess(String point) { + public Boolean canBeLess(String point) throws TypeNotSupportedException { return typedLessThan(point); } diff --git a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java index 826a36a1f..cbeb66819 100644 --- a/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java @@ -11,14 +11,11 @@ import io.whitefox.core.TableSchema; import io.whitefox.core.types.predicates.PredicateException; import java.sql.Timestamp; -import java.time.OffsetDateTime; -import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import org.apache.log4j.Logger; - public class DeltaSharedTable implements InternalSharedTable { private final Logger logger = Logger.getLogger(this.getClass()); diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java index 12d651549..6cfec32b9 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/ColumnOp.java @@ -9,7 +9,6 @@ import io.whitefox.core.types.BooleanType; import io.whitefox.core.types.DataType; import java.util.Objects; -import org.apache.commons.lang3.tuple.Pair; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "column") public class ColumnOp extends LeafOp { @@ -52,7 +51,7 @@ public DataType getOpValueType() { @Override public Object eval(EvalContext ctx) { // TODO: handle case of null column + column ranges - return Pair.of(resolve(ctx), valueType); + return new ColumnRange(resolve(ctx), valueType); } public void validate() throws PredicateException { diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index 0e9bba84a..6e641992b 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -2,6 +2,8 @@ import io.whitefox.core.ColumnRange; import io.whitefox.core.types.*; +import org.apache.commons.lang3.tuple.Pair; + import java.sql.Date; import java.sql.Timestamp; import java.util.List; @@ -13,100 +15,84 @@ public class EvalHelper { private static LeafEvaluationResult validateAndGetRange( ColumnOp columnChild, LiteralOp literalChild, EvalContext ctx) throws PredicateException { var columnRange = columnChild.evalExpectColumnRange(ctx); - var rightVal = literalChild.evalExpectValueAndType(ctx).getLeft(); + var rightVal = literalChild.evalExpectValueAndType(ctx).getSingleValue(); return LeafEvaluationResult.createFromRange(new RangeEvaluationResult(columnRange, rightVal)); } private static LeafEvaluationResult validateAndGetTypeAndValue( - List children, EvalContext ctx) throws PredicateException { - var leftChild = children.get(0); - var leftType = leftChild.evalExpectValueAndType(ctx).getRight(); - var leftVal = leftChild.evalExpectValueAndType(ctx).getLeft(); - - var rightChild = children.get(1); - var rightType = rightChild.evalExpectValueAndType(ctx).getRight(); - var rightVal = rightChild.evalExpectValueAndType(ctx).getLeft(); + ColumnOp columnOp, LiteralOp literalOp, EvalContext ctx) throws PredicateException { + var columnType = columnOp.evalExpectValueAndType(ctx).getValueType(); + var columnValue = columnOp.evalExpectValueAndType(ctx).getSingleValue(); + + var literalType = literalOp.evalExpectValueAndType(ctx).getValueType(); + var literalValue = literalOp.evalExpectValueAndType(ctx).getSingleValue(); // If the types don't match, it implies a malformed predicate tree. // We simply throw an exception, which will cause filtering to be skipped. - if (!Objects.equals(leftType, rightType)) { - throw new TypeMismatchException(leftType, rightType); - } - - if (leftVal == null && leftChild instanceof ColumnOp) { - return validateAndGetRange((ColumnOp) leftChild, (LiteralOp) rightChild, ctx); + if (!Objects.equals(columnType, literalType)) { + throw new TypeMismatchException(columnType, literalType); } - // maybe better to enforce the Equal/LessThan... to explicitly require a column child and - // literal child - if (rightVal == null && rightChild instanceof ColumnOp) { - return validateAndGetRange((ColumnOp) rightChild, (LiteralOp) leftChild, ctx); + if (columnValue == null) { + return validateAndGetRange(columnOp, literalOp, ctx); } // We throw an exception for nulls, which will skip filtering. - if (leftVal == null || rightVal == null) { - throw new NullTypeException(leftChild, rightChild); + if (literalValue == null) { + throw new NullTypeException(columnOp, literalOp); } + return LeafEvaluationResult.createFromPartitionColumn(new PartitionEvaluationResult( - new ColumnRange(leftVal, leftType), new ColumnRange(rightVal, rightType))); + new ColumnRange(columnValue, columnType), literalValue)); + } + + private static Pair arrangeChildren(List children) { + if (children.get(0) instanceof ColumnOp) + return Pair.of((ColumnOp) children.get(0), (LiteralOp) children.get(1)); + else + return Pair.of((ColumnOp) children.get(1), (LiteralOp) children.get(0)); } // Implements "equal" between two leaf operations. static Boolean equal(List children, EvalContext ctx) throws PredicateException { + var columnOp = arrangeChildren(children).getLeft(); + var literalOp = arrangeChildren(children).getRight(); + + var leafEvaluationResult = validateAndGetTypeAndValue(columnOp, literalOp, ctx); - var leafEvaluationResult = validateAndGetTypeAndValue(children, ctx); - var rangeEvaluation = leafEvaluationResult.rangeEvaluationResult.map(range -> { - var columnRange = range.getColumnRange(); - var value = range.getValue(); + if (leafEvaluationResult.rangeEvaluationResult.isPresent()){ + var evaluationResult = leafEvaluationResult.rangeEvaluationResult.get(); + var columnRange = evaluationResult.getColumnRange(); + var value = evaluationResult.getValue(); return columnRange.contains(value); - }); - if (rangeEvaluation.isPresent()) return rangeEvaluation.get(); + } + else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { - var typesAndValues = leafEvaluationResult.partitionEvaluationResult.get(); - var leftType = typesAndValues.getPartitionValue().getValueType(); - var leftVal = typesAndValues.getPartitionValue().getOnlyValue(); - var rightVal = typesAndValues.getLiteralValue().getOnlyValue(); - - // we fear no exception here since it is validated before - if (BooleanType.BOOLEAN.equals(leftType)) { - return Boolean.valueOf(leftVal) == Boolean.valueOf(rightVal); - } else if (IntegerType.INTEGER.equals(leftType)) { - return Integer.parseInt(leftVal) == Integer.parseInt(rightVal); - } else if (LongType.LONG.equals(leftType)) { - return Long.parseLong(leftVal) == Long.parseLong(rightVal); - } else if (StringType.STRING.equals(leftType)) { - return leftVal.equals(rightVal); - } else if (DateType.DATE.equals(leftType)) { - return Date.valueOf(leftVal).equals(Date.valueOf(rightVal)); - } else throw new TypeNotSupportedException(leftType); + var evaluationResult = leafEvaluationResult.partitionEvaluationResult.get(); + var literalValue = evaluationResult.getLiteralValue(); + + return evaluationResult.getPartitionValue().contains(literalValue); } else throw new PredicateColumnEvaluationException(ctx); + } static Boolean lessThan(List children, EvalContext ctx) throws PredicateException { + var columnOp = arrangeChildren(children).getLeft(); + var literalOp = arrangeChildren(children).getRight(); - var leafEvaluationResult = validateAndGetTypeAndValue(children, ctx); - var rangeEvaluation = leafEvaluationResult.rangeEvaluationResult.map(range -> { - var columnRange = range.getColumnRange(); - var value = range.getValue(); - return columnRange.canBeLess(value); - }); + var leafEvaluationResult = validateAndGetTypeAndValue(columnOp, literalOp, ctx); - if (rangeEvaluation.isPresent()) return rangeEvaluation.get(); + if (leafEvaluationResult.rangeEvaluationResult.isPresent()){ + var evaluationResult = leafEvaluationResult.rangeEvaluationResult.get(); + var columnRange = evaluationResult.getColumnRange(); + var value = evaluationResult.getValue(); + return columnRange.canBeLess(value); + } else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { - var typesAndValues = leafEvaluationResult.partitionEvaluationResult.get(); - var leftType = typesAndValues.getPartitionValue().getValueType(); - var leftVal = typesAndValues.getPartitionValue().getOnlyValue(); - var rightVal = typesAndValues.getLiteralValue().getOnlyValue(); - - if (IntegerType.INTEGER.equals(leftType)) { - return Integer.parseInt(leftVal) < Integer.parseInt(rightVal); - } else if (LongType.LONG.equals(leftType)) { - return Long.parseLong(leftVal) < Long.parseLong(rightVal); - } else if (StringType.STRING.equals(leftType)) { - return leftVal.compareTo(rightVal) < 0; - } else if (DateType.DATE.equals(leftType)) { - return Date.valueOf(leftVal).before(Date.valueOf(rightVal)); - } else throw new TypeNotSupportedException(leftType); + var evaluationResult = leafEvaluationResult.partitionEvaluationResult.get(); + var literalValue = evaluationResult.getLiteralValue(); + + return evaluationResult.getPartitionValue().canBeLess(literalValue); } else throw new PredicateColumnEvaluationException(ctx); } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java index 8238928d1..b7dc7e586 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LeafOp.java @@ -3,9 +3,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.whitefox.core.ColumnRange; import io.whitefox.core.types.DataType; import java.util.List; -import org.apache.commons.lang3.tuple.Pair; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "op") @JsonSubTypes({ @@ -19,12 +19,12 @@ public abstract class LeafOp implements BaseOp { @JsonProperty("valueType") DataType valueType; - Pair evalExpectValueAndType(EvalContext ctx) throws PredicateException { + ColumnRange evalExpectValueAndType(EvalContext ctx) throws PredicateException { var res = eval(ctx); - if (res instanceof Pair) { - return (Pair) res; + if (res instanceof ColumnRange) { + return (ColumnRange) res; } else { - throw new WrongExpectedTypeException(res, Pair.class); + throw new WrongExpectedTypeException(res, ColumnRange.class); } } diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/LiteralOp.java b/server/core/src/main/java/io/whitefox/core/types/predicates/LiteralOp.java index 012c31737..0fb87fd47 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/LiteralOp.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/LiteralOp.java @@ -4,8 +4,8 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import io.whitefox.core.ColumnRange; import io.whitefox.core.types.DataType; -import org.apache.commons.lang3.tuple.Pair; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "literal") public class LiteralOp extends LeafOp { @@ -25,7 +25,7 @@ public void validate() throws PredicateException { @Override public Object eval(EvalContext ctx) { - return Pair.of(value, valueType); + return new ColumnRange(value, valueType); } public LiteralOp() { diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/PartitionEvaluationResult.java b/server/core/src/main/java/io/whitefox/core/types/predicates/PartitionEvaluationResult.java index 0debdf19a..71031e935 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/PartitionEvaluationResult.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/PartitionEvaluationResult.java @@ -5,9 +5,9 @@ public class PartitionEvaluationResult { ColumnRange partitionValue; - ColumnRange literalValue; + String literalValue; - public PartitionEvaluationResult(ColumnRange partitionValue, ColumnRange literalValue) { + public PartitionEvaluationResult(ColumnRange partitionValue, String literalValue) { this.partitionValue = partitionValue; this.literalValue = literalValue; } @@ -16,7 +16,7 @@ public ColumnRange getPartitionValue() { return partitionValue; } - public ColumnRange getLiteralValue() { + public String getLiteralValue() { return literalValue; } } diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaShareServiceTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaShareServiceTest.java index fa0838dc6..90e2f8f17 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaShareServiceTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaShareServiceTest.java @@ -1,11 +1,11 @@ package io.whitefox.core.services; import io.whitefox.DeltaTestUtils; +import io.whitefox.core.*; import io.whitefox.core.Principal; import io.whitefox.core.Schema; import io.whitefox.core.Share; import io.whitefox.core.SharedTable; -import io.whitefox.core.*; import io.whitefox.core.services.exceptions.TableNotFound; import io.whitefox.persistence.StorageManager; import io.whitefox.persistence.memory.InMemoryStorageManager; @@ -273,7 +273,7 @@ public void queryNonExistingTable() { "name")))); StorageManager storageManager = new InMemoryStorageManager(shares); DeltaSharesService deltaSharesService = - new DeltaSharesServiceImpl(storageManager, 100, loader, fileSignerFactory); + new DeltaSharesServiceImpl(storageManager, 100, tableLoaderFactory, fileSignerFactory); Assertions.assertThrows( TableNotFound.class, () -> deltaSharesService.queryTable("name", "default", "tableNotFound", null)); diff --git a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java index bea163cad..ae9bf1024 100644 --- a/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/DeltaSharedTableTest.java @@ -8,8 +8,6 @@ import io.whitefox.core.Protocol; import io.whitefox.core.ReadTableRequest; import io.whitefox.core.SharedTable; - -import java.sql.Timestamp; import java.time.format.DateTimeParseException; import java.util.List; import java.util.Optional; @@ -74,7 +72,7 @@ void getTableVersionWithMalformedTimestamp() { var DTable = DeltaSharedTable.of(PTable); assertThrows( DateTimeParseException.class, - () -> DTable.getTableVersion(Optional.of(Timestamp.valueOf("221rfewdsad10:15:30+01:00")))); + () -> DTable.getTableVersion(TestDateUtils.parseTimestamp("221rfewdsad10:15:30+01:00"))); } @Test diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java index ce0a07fb7..59770b055 100644 --- a/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java @@ -24,11 +24,17 @@ void testLessThanOnPartitionColumns() throws PredicateException { var children3 = List.of(new ColumnOp("long", LongType.LONG), new LiteralOp("21", LongType.LONG)); assertTrue(EvalHelper.lessThan(children3, evalContext3)); - var evalContext4 = new EvalContext(Map.of("float", "2.99"), Map.of()); + var evalContext4 = new EvalContext(Map.of("float", "2.97"), Map.of()); var children4 = List.of(new ColumnOp("float", FloatType.FLOAT), new LiteralOp("2.98", FloatType.FLOAT)); - assertThrows( - TypeNotSupportedException.class, () -> EvalHelper.lessThan(children4, evalContext4)); + assertTrue(EvalHelper.lessThan(children4, evalContext4)); + var children5 = + List.of(new LiteralOp("21", LongType.LONG), new ColumnOp("long", LongType.LONG)); + assertTrue(EvalHelper.lessThan(children5, evalContext3)); + var evalContext6 = new EvalContext(Map.of("float", "2.99"), Map.of()); + var children6 = + List.of(new ColumnOp("float", FloatType.FLOAT), new LiteralOp("2.98", FloatType.FLOAT)); + assertFalse(EvalHelper.lessThan(children6, evalContext6)); } @Test @@ -52,7 +58,7 @@ void testEqualOnPartitionColumns() throws PredicateException { var evalContext5 = new EvalContext(Map.of("float", "2.99"), Map.of()); var children5 = List.of(new ColumnOp("float", FloatType.FLOAT), new LiteralOp("2.99", FloatType.FLOAT)); - assertThrows(TypeNotSupportedException.class, () -> EvalHelper.equal(children5, evalContext5)); + assertTrue(EvalHelper.equal(children5, evalContext5)); } @Test From 2460a95796014f78e6da397d27411f4a3fd8b503 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Tue, 30 Jan 2024 01:57:29 +0100 Subject: [PATCH 29/30] exception handling from supplied function --- .../core/types/predicates/EvalHelper.java | 54 ++++++++----------- .../core/types/predicates/EvalHelperTest.java | 2 +- 2 files changed, 24 insertions(+), 32 deletions(-) diff --git a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java index 6e641992b..f015b957a 100644 --- a/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java +++ b/server/core/src/main/java/io/whitefox/core/types/predicates/EvalHelper.java @@ -2,12 +2,11 @@ import io.whitefox.core.ColumnRange; import io.whitefox.core.types.*; -import org.apache.commons.lang3.tuple.Pair; - import java.sql.Date; import java.sql.Timestamp; import java.util.List; import java.util.Objects; +import org.apache.commons.lang3.tuple.Pair; // Only for partition values public class EvalHelper { @@ -42,58 +41,51 @@ private static LeafEvaluationResult validateAndGetTypeAndValue( throw new NullTypeException(columnOp, literalOp); } - return LeafEvaluationResult.createFromPartitionColumn(new PartitionEvaluationResult( - new ColumnRange(columnValue, columnType), literalValue)); + return LeafEvaluationResult.createFromPartitionColumn( + new PartitionEvaluationResult(new ColumnRange(columnValue, columnType), literalValue)); } private static Pair arrangeChildren(List children) { if (children.get(0) instanceof ColumnOp) return Pair.of((ColumnOp) children.get(0), (LiteralOp) children.get(1)); - else - return Pair.of((ColumnOp) children.get(1), (LiteralOp) children.get(0)); + else return Pair.of((ColumnOp) children.get(1), (LiteralOp) children.get(0)); } - // Implements "equal" between two leaf operations. - static Boolean equal(List children, EvalContext ctx) throws PredicateException { + // allows throwing an exception from a function passed as an argument + @FunctionalInterface + interface BiFunctionWithException { + R apply(T t, U u) throws E; + } + + static Boolean evaluate( + List children, + EvalContext ctx, + BiFunctionWithException condition) + throws PredicateException { var columnOp = arrangeChildren(children).getLeft(); var literalOp = arrangeChildren(children).getRight(); var leafEvaluationResult = validateAndGetTypeAndValue(columnOp, literalOp, ctx); - if (leafEvaluationResult.rangeEvaluationResult.isPresent()){ + if (leafEvaluationResult.rangeEvaluationResult.isPresent()) { var evaluationResult = leafEvaluationResult.rangeEvaluationResult.get(); var columnRange = evaluationResult.getColumnRange(); var value = evaluationResult.getValue(); - return columnRange.contains(value); - } - - else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { + return condition.apply(columnRange, value); + } else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { var evaluationResult = leafEvaluationResult.partitionEvaluationResult.get(); var literalValue = evaluationResult.getLiteralValue(); - return evaluationResult.getPartitionValue().contains(literalValue); + return condition.apply(evaluationResult.getPartitionValue(), literalValue); } else throw new PredicateColumnEvaluationException(ctx); + } + static Boolean equal(List children, EvalContext ctx) throws PredicateException { + return evaluate(children, ctx, ColumnRange::contains); } static Boolean lessThan(List children, EvalContext ctx) throws PredicateException { - var columnOp = arrangeChildren(children).getLeft(); - var literalOp = arrangeChildren(children).getRight(); - - var leafEvaluationResult = validateAndGetTypeAndValue(columnOp, literalOp, ctx); - - if (leafEvaluationResult.rangeEvaluationResult.isPresent()){ - var evaluationResult = leafEvaluationResult.rangeEvaluationResult.get(); - var columnRange = evaluationResult.getColumnRange(); - var value = evaluationResult.getValue(); - return columnRange.canBeLess(value); - } - else if (leafEvaluationResult.partitionEvaluationResult.isPresent()) { - var evaluationResult = leafEvaluationResult.partitionEvaluationResult.get(); - var literalValue = evaluationResult.getLiteralValue(); - - return evaluationResult.getPartitionValue().canBeLess(literalValue); - } else throw new PredicateColumnEvaluationException(ctx); + return evaluate(children, ctx, ColumnRange::canBeLess); } // Validates that the specified value is in the correct format. diff --git a/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java b/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java index 59770b055..1299305ac 100644 --- a/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java +++ b/server/core/src/test/java/io/whitefox/core/types/predicates/EvalHelperTest.java @@ -33,7 +33,7 @@ void testLessThanOnPartitionColumns() throws PredicateException { assertTrue(EvalHelper.lessThan(children5, evalContext3)); var evalContext6 = new EvalContext(Map.of("float", "2.99"), Map.of()); var children6 = - List.of(new ColumnOp("float", FloatType.FLOAT), new LiteralOp("2.98", FloatType.FLOAT)); + List.of(new ColumnOp("float", FloatType.FLOAT), new LiteralOp("2.98", FloatType.FLOAT)); assertFalse(EvalHelper.lessThan(children6, evalContext6)); } From e5721acf4facd0dde6ae84b298994113e3f25823 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Tue, 30 Jan 2024 02:13:06 +0100 Subject: [PATCH 30/30] updated jsqlparser package --- server/core/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/core/build.gradle.kts b/server/core/build.gradle.kts index 926cb1511..6bcd53da6 100644 --- a/server/core/build.gradle.kts +++ b/server/core/build.gradle.kts @@ -44,7 +44,7 @@ dependencies { implementation(String.format("org.apache.hadoop:hadoop-aws:%s", hadoopVersion)) //PREDICATE PARSER - implementation("com.github.jsqlparser:jsqlparser:4.7") + implementation("com.github.jsqlparser:jsqlparser:4.8") // TEST testImplementation("org.junit.jupiter:junit-jupiter")