From 6890bfd04a71c497322e8a1aaf8a827d6b426685 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Thu, 25 Jan 2024 02:25:14 +0100 Subject: [PATCH 1/7] getTableVersion, historyTable, and timestamped snapshots --- .../api/deltasharing/SampleTables.java | 6 +- .../core/services/IcebergSharedTable.java | 16 ++- .../core/services/IcebergSharedTableTest.java | 39 +++++++ ...3-4d82-9097-2fd746120aae-00001.parquet.crc | Bin 0 -> 12 bytes ...9-444b-917d-d77c5cfe5642-00001.parquet.crc | Bin 0 -> 12 bytes ...9-444b-917d-d77c5cfe5642-00001.parquet.crc | Bin 0 -> 12 bytes ...3-4d82-9097-2fd746120aae-00001.parquet.crc | Bin 0 -> 12 bytes ...9-444b-917d-d77c5cfe5642-00001.parquet.crc | Bin 0 -> 12 bytes ...3-4d82-9097-2fd746120aae-00001.parquet.crc | Bin 0 -> 12 bytes ...9-444b-917d-d77c5cfe5642-00001.parquet.crc | Bin 0 -> 12 bytes ...3-4d82-9097-2fd746120aae-00001.parquet.crc | Bin 0 -> 12 bytes ...9-444b-917d-d77c5cfe5642-00001.parquet.crc | Bin 0 -> 12 bytes ...3-4d82-9097-2fd746120aae-00001.parquet.crc | Bin 0 -> 12 bytes ...-b583-4d82-9097-2fd746120aae-00001.parquet | Bin 0 -> 418 bytes ...-25d9-444b-917d-d77c5cfe5642-00001.parquet | Bin 0 -> 419 bytes ...-25d9-444b-917d-d77c5cfe5642-00001.parquet | Bin 0 -> 419 bytes ...-b583-4d82-9097-2fd746120aae-00001.parquet | Bin 0 -> 419 bytes ...-25d9-444b-917d-d77c5cfe5642-00001.parquet | Bin 0 -> 419 bytes ...-b583-4d82-9097-2fd746120aae-00001.parquet | Bin 0 -> 419 bytes ...-25d9-444b-917d-d77c5cfe5642-00001.parquet | Bin 0 -> 419 bytes ...-b583-4d82-9097-2fd746120aae-00001.parquet | Bin 0 -> 419 bytes ...-25d9-444b-917d-d77c5cfe5642-00001.parquet | Bin 0 -> 419 bytes ...-b583-4d82-9097-2fd746120aae-00001.parquet | Bin 0 -> 419 bytes ...b8-a744-47c9-8f48-535f147b5df1-m0.avro.crc | Bin 0 -> 64 bytes ...9c-0fad-4fed-9671-579ecceb195b-m0.avro.crc | Bin 0 -> 64 bytes ...4929c-0fad-4fed-9671-579ecceb195b.avro.crc | Bin 0 -> 44 bytes ...c3eb8-a744-47c9-8f48-535f147b5df1.avro.crc | Bin 0 -> 44 bytes .../metadata/.v1.metadata.json.crc | Bin 0 -> 16 bytes .../metadata/.v2.metadata.json.crc | Bin 0 -> 28 bytes .../metadata/.v3.metadata.json.crc | Bin 0 -> 36 bytes .../metadata/.version-hint.text.crc | Bin 0 -> 12 bytes ...1c3eb8-a744-47c9-8f48-535f147b5df1-m0.avro | Bin 0 -> 6724 bytes ...34929c-0fad-4fed-9671-579ecceb195b-m0.avro | Bin 0 -> 6725 bytes ...-f034929c-0fad-4fed-9671-579ecceb195b.avro | Bin 0 -> 4319 bytes ...-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro | Bin 0 -> 4391 bytes .../icebergtable2/metadata/v1.metadata.json | 39 +++++++ .../icebergtable2/metadata/v2.metadata.json | 70 +++++++++++++ .../icebergtable2/metadata/v3.metadata.json | 97 ++++++++++++++++++ .../icebergtable2/metadata/version-hint.text | 1 + 39 files changed, 264 insertions(+), 4 deletions(-) create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00001-1-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00001-9-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00003-11-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00003-3-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00004-12-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00004-4-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00006-14-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00006-6-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00007-15-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00007-7-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00001-1-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00001-9-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00003-11-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00003-3-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00004-12-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00004-4-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00006-14-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00006-6-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00007-15-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00007-7-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.ac1c3eb8-a744-47c9-8f48-535f147b5df1-m0.avro.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.f034929c-0fad-4fed-9671-579ecceb195b-m0.avro.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v1.metadata.json.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v2.metadata.json.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v3.metadata.json.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.version-hint.text.crc create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/ac1c3eb8-a744-47c9-8f48-535f147b5df1-m0.avro create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/f034929c-0fad-4fed-9671-579ecceb195b-m0.avro create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v1.metadata.json create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v2.metadata.json create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v3.metadata.json create mode 100644 server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/version-hint.text diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java index cd18041e3..29a1f2dcc 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java @@ -41,6 +41,9 @@ public static InternalTable s3IcebergTable1( public static final InternalTable icebergtable1 = icebergTableWithHadoopCatalog("test_db", "icebergtable1"); + public static final InternalTable icebergtable2 = + icebergTableWithHadoopCatalog("test_db", "icebergtable2"); + public static final String deltaTable1Path = deltaTableUri("delta-table"); public static final String deltaTableWithHistory1Path = deltaTableUri("delta-table-with-history"); @@ -59,7 +62,8 @@ public static StorageManager createStorageManager() { new SharedTable("table1", "default", "name", deltaTable1), new SharedTable( "table-with-history", "default", "name", deltaTableWithHistory1), - new SharedTable("icebergtable1", "default", "name", icebergtable1)), + new SharedTable("icebergtable1", "default", "name", icebergtable1), + new SharedTable("icebergtable2", "default", "name", icebergtable2)), "name")), testPrincipal, 0L))); diff --git a/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java index ac6225762..0ae103fe9 100644 --- a/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java @@ -7,8 +7,13 @@ import java.sql.Timestamp; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; +import java.util.Comparator; +import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; +import java.util.stream.StreamSupport; + import org.apache.commons.lang3.NotImplementedException; import org.apache.iceberg.PartitionField; import org.apache.iceberg.Snapshot; @@ -59,8 +64,13 @@ private Optional getSnapshot(Optional startingTimestamp) { return startingTimestamp .map(this::getTimestamp) .map(Timestamp::getTime) - .map(icebergTable::snapshot) - .or(() -> Optional.of(icebergTable.currentSnapshot())); + .map(t -> StreamSupport.stream(icebergTable.snapshots().spliterator(), false) + .sorted(Comparator.comparingLong(Snapshot::timestampMillis)) + .filter(s -> s.timestampMillis() > t) + ) + .map(Stream::findFirst) + .map(s -> s.or(Optional::empty)) + .orElseGet(() -> Optional.of(icebergTable.currentSnapshot())); } private Timestamp getTimestamp(String timestamp) { @@ -71,7 +81,7 @@ private Timestamp getTimestamp(String timestamp) { @Override public Optional getTableVersion(Optional startingTimestamp) { - return Optional.of(0L); + return getSnapshot(startingTimestamp).map(Snapshot::sequenceNumber); } @Override diff --git a/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java index 2fdd76a29..bdb5fc30a 100644 --- a/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java @@ -30,10 +30,49 @@ void getTableMetadata() { assertEquals("3369848726892806393", metadata.get().id()); } + @Test + void getTableMetadataWithTimestamp() { + var PTable = new SharedTable( + "icebergtable2", + "default", + "share1", + icebergTableWithHadoopCatalog("test_db", "icebergtable2")); + var DTable = icebergTableLoader.loadTable(PTable); + var metadata = DTable.getMetadata(Optional.of("2024-01-25T01:32:00+01:00")); + assertTrue(metadata.isPresent()); + assertEquals("2174306913745765008", metadata.get().id()); + } + @Test void getUnknownTableMetadata() { var unknownPTable = new SharedTable( "notFound", "default", "share1", icebergTableWithHadoopCatalog("test_db", "not-found")); assertThrows(IllegalArgumentException.class, () -> DeltaSharedTable.of(unknownPTable)); } + + @Test + void getTableVersion() { + var PTable = new SharedTable( + "icebergtable1", + "default", + "share1", + icebergTableWithHadoopCatalog("test_db", "icebergtable1")); + var DTable = icebergTableLoader.loadTable(PTable); + var version = DTable.getTableVersion(Optional.empty()); + assertTrue(version.isPresent()); + assertEquals(1, version.get()); + } + + @Test + void getTableVersionWithTimestamp() { + var PTable = new SharedTable( + "icebergtable2", + "default", + "share1", + icebergTableWithHadoopCatalog("test_db", "icebergtable2")); + var DTable = icebergTableLoader.loadTable(PTable); + var version = DTable.getTableVersion(Optional.of("2024-01-25T01:32:00+01:00")); + assertTrue(version.isPresent()); + assertEquals(1, version.get()); + } } diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00001-1-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00001-1-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..88849bd70caa27656ad905c31217210c435373fa GIT binary patch literal 12 TcmYc;N@ieSU}Cs++^84;6chu2 literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00001-9-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00001-9-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..c2a62a9e715f5aeef8512fe2f97e1a4abca6ccfd GIT binary patch literal 12 TcmYc;N@ieSU}8A=`~6J-7EA=k literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00003-11-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00003-11-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..36b4662e7f0b2f00588faf3e0c859a3f5fc39cd6 GIT binary patch literal 12 TcmYc;N@ieSU}DhroudZ;5P1T9 literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00003-3-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00003-3-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..cae6372c5aad795300bafa697ae666bde6a801e4 GIT binary patch literal 12 TcmYc;N@ieSU}9MH^Vb^y74rnd literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00004-12-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00004-12-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..0c4d58486dce94c12c3a5ee54c9094a1298d0400 GIT binary patch literal 12 TcmYc;N@ieSU}AW9$WtEx6bl1Z literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00004-4-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00004-4-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..0e006ee21ef4caf8dcfb2830ac2375d67569e152 GIT binary patch literal 12 TcmYc;N@ieSU}9Lm=4lQ96m|pW literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00006-14-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00006-14-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..5060ac6bcc298f4beac44bd177c2f6a8d132c403 GIT binary patch literal 12 TcmYc;N@ieSU}9h{vAO{O5HteB literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00006-6-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00006-6-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..eed31addd79dbce7a30b24b78737877adba09943 GIT binary patch literal 12 TcmYc;N@ieSU}Er8%5Xl15 literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00007-15-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00007-15-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..a1fce5513fc4e5f93999afcd017f8dd7e566c256 GIT binary patch literal 12 TcmYc;N@ieSU}9iXRE+=t4xItt literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00007-7-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00007-7-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc new file mode 100644 index 0000000000000000000000000000000000000000..b72dd1703de8810dd8e96d97dfee0a31733b9f6c GIT binary patch literal 12 TcmYc;N@ieSU}9*W>O2bo5_d!USe91X?Og>xAi>B6K6-q7*?9q~OdX*1+^N4?#-5z^y;w z2l!`h1v3;|@9I2~b1unbbSnsuQ^Lvn_ve#MkYSi$fcn?P4?K)uKkN)K%NV=)2MJ;n zfY}t2u>oMXdwrAzVXO!OQm%r9w7f$=5wn?H!!{|iDh5`_NcZLVaI( zP)(Ux(-Z;Fj}ocSdvXaT1=TPeSBhP%6CJ5gC9`H4y{LI$O*cGyTPZ%^X_n+s#$CR1 z8d~y!=kjN*Vwv*6L+ZZdO6E&Q;c! zC(f%%QoUR_qAC2QaOz>MW9i7SC3?Q_`m!B%{g#T_eczM4t{2HbiH=Y$uP0>~oHN@t MuXt4gn3&)B1v%VNGXMYp literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00001-9-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00001-9-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..60840c57bff8f9d268f23aa3f2267d42f52e6b61 GIT binary patch literal 419 zcmYLG%SyvQ6uliP45FgY88VPX7-*@G)(lDQ11WSP;wlJ2LAppXV-2LQJOn9S`49ey zzv3785pD!06q{L{bMC$8T<-Xj2SI?G6HdN=zTQtwf;7bh1GGP{eqlNRIEMXW7l;}7 zz$!=(qX5jLnD`oi=JwrD9)yV^2uQsN28!h)0*aW)%oa9Doi#DAQYI%9(J}Q~_@;8K z5QX+3vQl-JUQ-7F(2p8v&?y;%c0nyn{YJTqaTZ4^RO!5%MK5X*7_%+U-`0wcc$TL{ zlyjT!E1g*Kkz@1cSS2#!qo*w|6=CybuqgXAvw5mEMVu;K(9_EP_exgFIp588VBD{R zbW^Cjvq-I1m1gm3X^F0IyTWRRMVv@WhMpL>!Wqh5*mpe@^@gq^2Yn}!ffCn3dCow} RFt}u-wl%>O)L396Q6o({BNNarsEw)LEO#-^;z)%mUG%c(MJ)qkzT^4ZMsbg4d0Ip{ zxA~#cjV1RSn?J`Yks0@%cDz)C%~!#)?AOfZsoEBCs&qk5EBoIoS+5p+Ki`9KzX{T9 zq4MrBwO&=4#p{(Ndcy4qs}mM+A}tyA#mE)TSPsIW+gH(G>^gEZbRrojaV=Ee8A%xi Pm&`Qv6>n<*zW$y+Jhf5m literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00003-3-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00003-3-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9ab441824b19ead8c7471a77882acb54b32e5f6c GIT binary patch literal 419 zcmYLG%SyvQ6ulj)%tAz=Gh`r(FwjyVtr^nV2U6%p#8nW4f^?B&#u`XlJ4pm7-3tDI zf8sy*1AdO+gkm$RbI!f@oXeeHdM^l&Gs4OD$NSH*O^~6OV1V}5BBdAIEPhtY(3)>~{`R8ym}j{z z;+(sDU+E;0k3E+^B`TE}A3tt+sR);^!e!ZSn9H@=6p2=*V5XJ(@0G4s3%;B0z`A=0 z^`=mHXQ`c6rL$zcazt17UE#E&B1xqqqn;T0!W+qcH1K;W?vH#=4hLQ=LnW?+>Ul#c Rqws>+wz=YsxpZK@=MR=+QE~tP literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00004-12-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00004-12-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..68b37ff08aaeb0962e60ccf2aa06b234ef1019fb GIT binary patch literal 419 zcmYLG%SyvQ6uliP3{pg)Gh`r(FwjyVtr^nVv`C>F5m!MF3erWA8EYVYL396Q6o({BNNarsEw)LEO#-^;z)%mUG%c(MJ)qkzT^4ZM)5Jv^0bI@ zZu3K>8%sWRZ2lapL}q;awBw~BY`zMXWxr-NPt~@FQ>6=fTG{_z$$GWm`}rP>`%RE; z3zc`5sr9PTEMBiH(GzY@Se>wl6KTn?FFaQ`BRL3%ZeK-%k?Tlr=tMG5;##P_<4G9? Pm&`Qv6>n<*zW$y+=a5mj literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00004-4-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00004-4-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..00a304e2dd4d584dff0c56a9cf27a3864cfb23b6 GIT binary patch literal 419 zcmYLG%SyvQ6uliP%tAz=Gh`r(FwjyVtr^nV2U6%p#8nW4f^?B&#u`XplSGixg+Jk+ z_yhipOFuwxLa~|EIp^Mc&gD)peGml5IpO5z`~3^s1R076255g?{laVtfZ*`h1!5LH zvI-K!C;+o5CV>H9xO;b!hf%5s0#a{+g%bIQfFfoyyJb@9tcih@GC8G)PH51=HJ!sV-QS@s*|a;>&SqLnF_Y32TVrR&v#@8^54?l+;{ z7Ao&7wezZUmaJEf=nB6poOV8(5#I;a8ZzyFH QUNYM@SG=tO1m=7G09E@@>;M1& literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00006-14-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00006-14-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..78377a15db342a38d8dbfe0daecb5efc1be02696 GIT binary patch literal 419 zcmYLG%SyvQ6uliP%%X}yXUISnVW6c#S~H}zZID7YBCdiU6r_tJGuA-*%0rOS-|#nF z`75sc6~zg~W>)8%d+#}yJO1=R5FqD-ldtdJkA_K*rkG%W&gaz+%%%XRaCqzjF#{i2 z1qoslfSD8%UjxwGzB|c-Fi`{nskgvDv3x{85i^@ z=o}*}Rfp*{br1l3uaPF5kqKxQ)W+0rmb(~dail_(E_zw?qLzU%-|_ryqqxVjJT0P} z+x$@J#*%xE&7WhH$c%eWJ6?_G@PIRBekmRl1<3mHqFPtXB)ZpYOrA-vsHl zPm9*)d`C@k(LbmV&n>EEC=Dx?W<@ob{#nyI*|;NxE8AKjHC>M POJ)}yJR7Uu+UN=ty$9AHb|it5l=x73erQ8U27oy%7-AO|3ds3 z9t3}zC&3NHW>4?EnR#!R^`{Sl068a|e7%4EG)#gt#RLO%zOQ~^HU&6=!($hS8TiO5 zND!j{%%qt38i3~Z-Dw_#i6RI{y#)q}Ihy;|`7d=JL`CP=r1 z%Dc&Vg2iDaO}wNQO$BxM*} PGSk#oysZKF`g{HW7ll#B literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00007-15-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00007-15-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..7d3f007f72586100f99e4b0325c71c5b17280b9d GIT binary patch literal 419 zcmYLGO-sW-5S=a+_8>)}yJR7Uu+UN=tyxm-2U6%o#8VK2g7lDN*BVH_Hi;mmf5U$w zcS*zD=OH#6@Iv%&O15FlrSlkcyePh=BhC?*)7^KtnLvnfCW4v(E9X5k~N zAVG`*Fq>i$7yyR5cPDumrHUXR^%httk&g%{Vm7ne*d}$>!oW(IoKi$5G-%_S%CSNe zI>*RLHDP8=69hotYotkMWCF$owJ{Bvmts% z%MX=qBKg>J`E#ODnep+{j+csX`6^tN{f4<*t8I~JWeR3mx&L12dbQyD`5vtMO{lkp z%DYSLyegd~>y;yV!tV*E6BS7+9U1k-&==lFUPS}Hui~qb@5$l7i)E<9wNQO;C}k90 PFxxg)ysZHQ=6n7C6&+G} literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00007-7-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00007-7-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet new file mode 100644 index 0000000000000000000000000000000000000000..6377a59546c2f130bd1d8e6fc5962db054400169 GIT binary patch literal 419 zcmYLG%SyvQ6uliP%tCxXXUISnVW6c#S~H}z52VnIh^rt71?eKmj5Uz9c9IBE3aZ{ef*r-1R076255g@{=#euP>20v=ZIPO zz$!=(qX5jNm;?rZ;qL8G9!9Am2uQsN7E0s;0*aW;>=w33oi#DAQYOa~(FqM&__}he z5QX+3vQkZ$S5M4aX~FigGRZFb&|v?QhL$N;%BuCt@)PcZ!e0Ed6w%U z&biC?l};l0*mL<)qEeah@#B`4ig5WVT$cTYxm>GFk!WQKW?H%bUg>(Z;Jf(_th<*` zZwi%nmfCq$I!o3oM|6eX6;3-Ul2ke}>WQH*ypilj1HY%@{>b;_aNxx- Q3NM&#n=9Vb00Q$pf0Bq%bpQYW literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.ac1c3eb8-a744-47c9-8f48-535f147b5df1-m0.avro.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.ac1c3eb8-a744-47c9-8f48-535f147b5df1-m0.avro.crc new file mode 100644 index 0000000000000000000000000000000000000000..73a65f1f7e2a9f9130a991790d5fa30cf27861cc GIT binary patch literal 64 zcmV-G0KflZa$^7h00IDB-S;4|ucIjNNL*s^kP`jg#IoMZRR?)+7#olBf$v|f0TdH3 WLh45~s8mIrbHF1-`W8|5AcPo^xEwwJ literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.f034929c-0fad-4fed-9671-579ecceb195b-m0.avro.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.f034929c-0fad-4fed-9671-579ecceb195b-m0.avro.crc new file mode 100644 index 0000000000000000000000000000000000000000..f1f1d1e1d1146c13f823f9fa421e03781d39d81d GIT binary patch literal 64 zcmV-G0KflZa$^7h00IDB-S;4|ucIjNNL*s^kP`jg#IoMZRR?)+7#olBf$v|f0TdH3 WLh45~s8mIrbHF1n62$6;=L>t0!yLx| literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro.crc new file mode 100644 index 0000000000000000000000000000000000000000..d74c496a0f1bed9cb10e7141852312f88beaf809 GIT binary patch literal 44 zcmYc;N@ieSU}7ja5>(lm9K;mLKe58fI_Ag%kEtAfo@+x`ymaQ*?0@s^_@_;r075+x ALjV8( literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro.crc new file mode 100644 index 0000000000000000000000000000000000000000..d60fe28ce0c01202703a3392303decfcdd3575a0 GIT binary patch literal 44 zcmYc;N@ieSU}7ja5>(lm9K;mLKe0mKyr;_okEtAfo@+x`ymaQ*?0@r(t$*f306U-% AWdHyG literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v1.metadata.json.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v1.metadata.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..3ff5d1018aca8c1a7192dcbfc1a4e297d3b5c1bf GIT binary patch literal 16 XcmYc;N@ieSU}Csx>-Fo`LS04xCxZoA literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v2.metadata.json.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v2.metadata.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..0b5ac8526d0f1edda36483f9b4f790d6cbaa9393 GIT binary patch literal 28 jcmYc;N@ieSU}E5M;#XoQh+tE6N;c)&7ZJsB>An;IP0I$( literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v3.metadata.json.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v3.metadata.json.crc new file mode 100644 index 0000000000000000000000000000000000000000..5c44f6ebb1f74929b4b3dedfbdb674f03782e62b GIT binary patch literal 36 scmYc;N@ieSU}BhlO~&u%Y(W=Ar)1M-9jUz&>I2yw-noQNIO!ZXRAb? z$T7*pS$ZD+i%B+l+0$(TFW@$b1+omw?31?jZo3Z;*w$mdz;~KMRW{Kqn_kb0%3@Q` z5XVgt)3pcTxu7(E#Bs6fIq)iPt!`3N**V|RZLzI^ILE{}r{8ju%tZe}k7Rm=0auu5 zy)h|GGT1X*DM6o$l4Ma5xJOKlBuy`yB6g_0a%rj{Eu$MG=}cKcI~5aTEPmkzQr$n7VHYD%DZ2UQzGa!1Z`bXT`blqLpo zi4#Lg2l$5#-5m^`1zK5{WL31D0g}URKi3Xfs6*~Re3$`iEDbXS8`rHuTXeS{tyqs( zK2eW(_~m$9K!wT8Sauc*enlj&09TNI22m&yh~{bxS+++1pd7X#_Jv6%$HnA;6E5>U zw7xXI%FRoRHg~(b2xrr)${Q;5XAO&O!eR(0Cf>DAvA*NKYfzf*(bQfkW#@cQ%7X zrJGW`Eoze=;OT*?9iUR;)j-uAHH+h#&o}`pB|ZhHSU$%flQhFiLythTaU*P|Q5Wti zkDX{Fy_IA^okjo^nF3f);^8zZpKFTPQSO*)ic%GT3@IiYa?tMzr87+yJE@7yX$MPf z1F+QgDzZ(TPz2?&O>M6dEIR6Gy+U3ghFaJj{R%FdY^vBi3~LLqlPSO)R;ZxiOrfVK zEi}(stAI{vDP5>VIj_UR@cs$ zzmPuEnpf#N>-VU6%|r6u8)T) z%zmh@;@ZlDX&UGmw~1$PgqRgKE9jp-|1D3{xDuXFewIflS|zGVQZ}xoEv$kFS>y5x zN;!m>(xI3Hv&bI@ax43Xenesvm5ti`E70Lj-b8?e|CXzuwFZRC2dF}7#eHZGePD7q z3E&DU^8p0W)xL-i3KZ7#_)wnngigu4l8e!sNqVM0)t2%M3T&#LO&}wo8!Dtle!djp z9h3Azg+wahg$kjYx)*n3387y~bLdxu5KPi1rKO}#3Sbxbl+sPiKBKY+jaTv5p=rAtN?N~U`A#mzc2U*$sx@^9BS@YxexoJ;5l zyQ85y^gCnWf(m0J{{UJ{{!>bDf(tB&XpzmDo5yo7vP5%X#e-PXI!HurUEzMsQx30VCGB>04`mRL7s}Jsd zZ~vKx-fHiDVD_Qf%NOOYQ}Oz`moC(Fp4c;T?v$7NFtcpd&F!N*_i9Vf>Z`Taj<(IH zxl!LxCm*+GKYMyd&lAenzH{%6k6pig9!u|H!t^&d%OV&d!W8vnS_q zNcIDc5-B^hv`Dd#$mj=J^utKcsE9HtDwGr&(bk!a$qr5-4U-0^DO=uHb&&jHu5E1-0^?FKZi$;XBt{Ev;Xxq>) z;3r~4FqI5G5C263N86m$9SvTEzdQ+gfiZeQTDjZ~GRsg{GU=R{=` zQ%`HIm(Wbl>4WE-(%cc(Q@pGTuX5JvCN`Cv^DP~S^atWxQ?cD1%S#Xw{R7YkYMCCuPVWmovMEc27%%r?^w%eAgd%9(! zlxAq2=0=dz4*n^I?)3%F0<9!WqRd-Q0I?OXhiQi_)FF2uKE#0gEma{39(u3?ZPC3R z6kCf(K4Fhp_{C_HLxspqTTVvd{0d2qfh)*AgD@0r2dd?0Y}fj|htm?AH08(PHoP(ew>)lvxH#n@AI$h9>!S3F-s_D2W)ryb`C=sC2GLd`G!vu1Qiw01~8#aL7TwD-_Q(k?*9Y zVoo?%atnYZw^opC@~9#xm2Glsg<#=PkLwlk3K7(3d-N-~WU|S8^DwL}nv+NZ=CDEq z4QCuZO=`k<)?5K}Qd99lEzp^z=8CW*{vzUUNsy#Q&YG-kPb^5do5_`QGl_2$p<*lm zl3ZCkBmR~H5})nRMG_g!Q}C8BDw71?I67}unyS-hlf4PD$Oelgj_+mXz#Ho~3aq7sZG0NLmIG)xDahqQyaO9S1GE;-!jcJRK}Sgy4StTX zA&MtJEumGamD1E|u+C6CZGpea*7s_T3k&%aNuc{;J>C}~4pX8FbX9}i)}3QyU)0g{ zsXX!rbTCZEiV#5TEiUnV%9u~g*wwURxMA{`yDT%6qli;k0vf0Q!U{&yud<{_R0_|y z*j1kWjs}%PkK@oXGm6o#!ont=j?u70SvUzaIU-iXju3q+Ma-B6_O*a$pp>jU9kDVg z8q>$ZM6(~NtFX2bVVVGX#%y9493p0g%?kRb&woo3HKv3ml%M4xidK%QqLhtkX`xjR zB5O>3UMW+EF&&BsFpKB~ZVlM(S6H5RB0$si~+>5?~kjl+;ayd`2Y?>Y0*j zRt~?vGxJ&5hgNkaxvY|rikFPcl}zR4^P4p_UuHuH@_}EXi# zd3nZ_-mi4e{#S)^$Bz<|ZAP`sHi$ zMy5^a+OxCgxkZ=f-*9m1cc+$io`}{?x4%C;d+r5c*2SHB57yjR6Fso~m!}`zH)BP| z-i`O%)P26zh__7NUT4pY#r7RoH91y0xa9M%ufA^myZP#skz-rqAGO~meY)hmJ@Ze; zHg-Jm$;v0UzO;L2zxKwk`26wpNA7wHMc+O2=gA%R?+f+)i&h`+Ss0n$|H8R5v0Za6 zt=qBw(oo&1jt|I{*Lx literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro new file mode 100644 index 0000000000000000000000000000000000000000..31e7e930b2edf3287394b380387e00e1e95fa469 GIT binary patch literal 4319 zcmbVPTWB0r814qe5^Ag>En=*v|wGLvMpo0TYS(n26=aqB}3bvQe7COdU@W;=5x z>ox=}eeu#FKD0``P$7s?E%8OHph8d)R1ooog*@a%t4~$zo78jWa?Y9Enayqw7hLB7PphIDd~4UDgqw9Fl_VEwj;l*Uk_~J0cMaO6fjq1lriI93 z!$g+uNu{~Pwi9)TtK>u!$iuE_&y(M}qmekXjs~&m*x-rBXF$WmRdApT0(2I2*wm#` zUY3`J+GAVDo)@EAju88hgP{dzg1|G6BVgK~(!|J%(~SZvLf|@du}Sk$y*koVg3J;( zZOpPRlRv@CH1Iv71MGl?YvR^?M})ANV`CJ&A9XX}>t5BtiGd^wi%!s~;3~vn4INPA(KZy? z>|mcgRdf8TaqXqe0c4tP6U4`aM#*DBT=b0TVY2ROccJ%3$2&w9#}F=s-evIVV1+tm z-rjfDX|(ks(e|(z^jKOi6!8wQUTheYwIMUEIJ>wD8-|S=kQr5+93ukC7U89xlWgCz zB*H?`C5EmGo8)hD_iKdu*z;Ddbn|jCUhxD)Qd)qf?U6f61IKYIj$>>vXLMLmIV$35OfkHb4Q%45$Ei9F2PD$P< z3Mp8G4P3QC?iT0OxT{h~%?$rw%)}sg3j}Wg!*Ky5Wm8h8lj#5~>V;pgZvr@Yt9M{! z>B!QML~dJ+kVK?0#U)l)*{{_g;SO2~f?AcgvtvDSfrPz15NFNNkv2d|w;)D4Vs0cz z_m3!ru~B(^LK!WLd}-rv~xx0&+c~YkecTVlo~MHK0!!G&MvdV?I*-pYLXy zOvKvPsO*J)ASgyoh<&8Z?Yas?oWuKw4*Tn&%jBLRi+={ z5F-qNT`a)~WPpfbp6(17fKp%p+!J&Wwil1U!SW0&o;i018K8-Vm3wdt9Ce(bru&3$ zkh}B^3{)OvplObw8_04O4ua#2wcw=)dNa2!6aSV=br z6U%EVio!2eIG==CXLKsDi)~JWRiN0C9WL;>(;B80}DX z<9#up=ZUK3Tf_yceI!S z8~75Any8r>l#O6h{0T=T+7*4jCap*&sA&_arn($_tajr`I2f0!r2Xg_EZc`9oCc3e z=S3_?m~;ofL0-R?EF85df{WNnP285fssk@Rbk9vssUQ7*>Z{8}-&$Y(rEhLtmA6lS zFnHxlYwN>@*Ps96xzXQ#y>n&#i|Q{gynABL$H(5j<;n9~`p@6+ZO%M!?v78JJ6CpA za@&6Fzcz5`pWXeZ*H+IiKXP?jZte$n-?jg49htoCMeE5m={gwSwkG}rX@poQ*dTLxXjKhCj gd}ICVtAC!^^Wx=yy=T|dm%p2S@BMvCxz-B)2W{`hXaE2J literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro new file mode 100644 index 0000000000000000000000000000000000000000..a1b42d070ffe88b905beaafd38eabe4d62aeea97 GIT binary patch literal 4391 zcmbVPTWB0r7_POb^-`KzBGM3#LyeavIUjRCT`EpoXt#`omppQ zvyCBO2zUwJf~W<-kVYyZ78NCZ@}X*_g+MK|MA53K6k~0%q7{nbnVFq)&g?C_dED7E z|M%a{_kFV`z5kkZ&2Yrf?>9vmW#L<`W1KabLtKnAkf<9H*Uo7$YyZwdP01kBN@Wxk z;l~*TsglXX_KuaE$U#fSCyIeg%TTmo{9DpR94DcR&{A{_Jl59(3W_Cz-AUk}lc*hv z#Kl59Ki*y*TSeM18(r0z*!{W%RbVFwOywyA6b+76S`R;2bn8#m*;X=8%}e z$Ru$^vq;uS{3qz?26+=nz|uj%P^{8?158*^*DU0`U+e-KY^VFD3G@uGWCXm{#sD9e zpat835SNgO$}$%s41|fE%p*-iDJ`E(BZKDcYe}+r3b?00K}MPn6cnHdpd=x?a%ZS+ zmxK_5l~r(m>cUIt1-vA{JM6*YNwAfcK!q77CzX_`!JH}UmTw^b!ip#CsaP_!+!|`2 z@WC<^yV=e@n<-JxR~y$3yEy<)(-;NbF{V*`ClePPP)rN2d#b+ByNbtaL}$m~E`?NQ z@NQ>?YGq#DcSA3f^}^S7w;A@aO1%)+J3xA|Y*5mMs&U!bd0kjGtk;05QQ656BCy%Q zy|jBdEw8E^W})IGhK2!0@!$CFXEF7W=cQa}g!rhp;y#MBX#pyliSMi(*o&LibrnGk zKX5s+-^gR~$^fX1x@;!fcNE_xRRttZFap(lDF<~iyGX&D(Ege@4|8ZaP^eYFlu!od zRf~(+Ir(`b%A}xTHZW3Ye77jCdR^sQY9{!-9uouSE#SNbGP(gEF54z`doneES-r^T zaw`CK-fA3J7#|vM=kRUIVv;CoOmv(Smdt0f5OarJ3WB0c+gY<7K7zw;bi`TIB_yuH zrK`|F%^o*mq_$u%5{d-7f)PFv4fD}ZG!SjUI9X_Qj3C3ry0$vd>XC7Ws{yM6EsBV+ zWDFNo|Lh%Pld)LmMO*ewJsQLz$3B|x_62qYoS@8y3fW1F1icpS1 z^uG#@kF2Wt0SXZy5aePpPM``1E9RA*0Ra#q1b}*iI>I*M5hz%QfO#{g?w|^&uVKjs z+yX@n6V%E+p&sNqeFFjIg9NmaW2gr*S%-t5c%3D96$HKtD7!7?UVV6 z*0WF?IxbI1R|ezBYef`+UMzP$KDDmWDc>%ZISo>QKxa=n)o^&yB*q<31UhZ6M-2mb z?Wq)>r>sC|$5uDmmkd-*!k&blRqQVDu@b2v4;V{Ah@AOOb4j&}EHX6+t{b7~KI-u+ z`r^KK7PU<;O>{rJ_TRsAYZB92!NMDv-Is2=)OW0{@42Vr`KI~i&wjWnoc{TiZy&vI z`sifq*7P5@Pkz5`$Kktg+;{w;^^+6b!gp79eslfQE5VDCC*F*_eXrE=?9j}m?H9gU z`eml~%E!-4Y#2z~ARm3VXa43*cOH~qNWXt;$E%-MgPj8dUw+*?^wY-Ua}P|NS^6hF zv+djypSB*E;`Tf~_}}wW@9m#Gd-8*mv*_3~S~~n=B0QBilDq4#LxJJfK6-Fr`}r$> Op8xIlLzizVE&YE)Ea>|H literal 0 HcmV?d00001 diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v1.metadata.json b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v1.metadata.json new file mode 100644 index 000000000..3c1770f16 --- /dev/null +++ b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v1.metadata.json @@ -0,0 +1,39 @@ +{ + "format-version" : 2, + "table-uuid" : "77734b3f-6c0a-4151-83b9-9be5cf0197d9", + "location" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2", + "last-sequence-number" : 0, + "last-updated-ms" : 1706142410682, + "last-column-id" : 1, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : true, + "type" : "long" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : -1, + "refs" : { }, + "snapshots" : [ ], + "statistics" : [ ], + "snapshot-log" : [ ], + "metadata-log" : [ ] +} \ No newline at end of file diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v2.metadata.json b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v2.metadata.json new file mode 100644 index 000000000..6d42ec518 --- /dev/null +++ b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v2.metadata.json @@ -0,0 +1,70 @@ +{ + "format-version" : 2, + "table-uuid" : "77734b3f-6c0a-4151-83b9-9be5cf0197d9", + "location" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2", + "last-sequence-number" : 1, + "last-updated-ms" : 1706142720036, + "last-column-id" : 1, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : true, + "type" : "long" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 2174306913745765008, + "refs" : { + "main" : { + "snapshot-id" : 2174306913745765008, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 2174306913745765008, + "timestamp-ms" : 1706142720036, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1706142687377", + "added-data-files" : "5", + "added-records" : "5", + "added-files-size" : "2094", + "changed-partition-count" : "1", + "total-records" : "5", + "total-files-size" : "2094", + "total-data-files" : "5", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1706142720036, + "snapshot-id" : 2174306913745765008 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1706142410682, + "metadata-file" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v1.metadata.json" + } ] +} \ No newline at end of file diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v3.metadata.json b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v3.metadata.json new file mode 100644 index 000000000..30dcb3b4c --- /dev/null +++ b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v3.metadata.json @@ -0,0 +1,97 @@ +{ + "format-version" : 2, + "table-uuid" : "77734b3f-6c0a-4151-83b9-9be5cf0197d9", + "location" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2", + "last-sequence-number" : 2, + "last-updated-ms" : 1706142746160, + "last-column-id" : 1, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : true, + "type" : "long" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 3375739370786085828, + "refs" : { + "main" : { + "snapshot-id" : 3375739370786085828, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 2174306913745765008, + "timestamp-ms" : 1706142720036, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1706142687377", + "added-data-files" : "5", + "added-records" : "5", + "added-files-size" : "2094", + "changed-partition-count" : "1", + "total-records" : "5", + "total-files-size" : "2094", + "total-data-files" : "5", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 3375739370786085828, + "parent-snapshot-id" : 2174306913745765008, + "timestamp-ms" : 1706142746160, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1706142687377", + "added-data-files" : "5", + "added-records" : "5", + "added-files-size" : "2095", + "changed-partition-count" : "1", + "total-records" : "10", + "total-files-size" : "4189", + "total-data-files" : "10", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1706142720036, + "snapshot-id" : 2174306913745765008 + }, { + "timestamp-ms" : 1706142746160, + "snapshot-id" : 3375739370786085828 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1706142410682, + "metadata-file" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1706142720036, + "metadata-file" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v2.metadata.json" + } ] +} \ No newline at end of file diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/version-hint.text b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/version-hint.text new file mode 100644 index 000000000..e440e5c84 --- /dev/null +++ b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/version-hint.text @@ -0,0 +1 @@ +3 \ No newline at end of file From 89959bce5c6845fdd8f5218ea82819a45ed6b3c7 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Thu, 25 Jan 2024 02:29:14 +0100 Subject: [PATCH 2/7] formatting --- .../api/deltasharing/SampleTables.java | 2 +- .../core/services/IcebergSharedTable.java | 7 ++---- .../core/services/IcebergSharedTableTest.java | 24 +++++++++---------- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java index 29a1f2dcc..a7a069505 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java @@ -42,7 +42,7 @@ public static InternalTable s3IcebergTable1( icebergTableWithHadoopCatalog("test_db", "icebergtable1"); public static final InternalTable icebergtable2 = - icebergTableWithHadoopCatalog("test_db", "icebergtable2"); + icebergTableWithHadoopCatalog("test_db", "icebergtable2"); public static final String deltaTable1Path = deltaTableUri("delta-table"); diff --git a/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java index 0ae103fe9..f55e2dafe 100644 --- a/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java @@ -8,12 +8,10 @@ import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.Comparator; -import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; - import org.apache.commons.lang3.NotImplementedException; import org.apache.iceberg.PartitionField; import org.apache.iceberg.Snapshot; @@ -65,9 +63,8 @@ private Optional getSnapshot(Optional startingTimestamp) { .map(this::getTimestamp) .map(Timestamp::getTime) .map(t -> StreamSupport.stream(icebergTable.snapshots().spliterator(), false) - .sorted(Comparator.comparingLong(Snapshot::timestampMillis)) - .filter(s -> s.timestampMillis() > t) - ) + .sorted(Comparator.comparingLong(Snapshot::timestampMillis)) + .filter(s -> s.timestampMillis() > t)) .map(Stream::findFirst) .map(s -> s.or(Optional::empty)) .orElseGet(() -> Optional.of(icebergTable.currentSnapshot())); diff --git a/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java index bdb5fc30a..660a57888 100644 --- a/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java @@ -33,10 +33,10 @@ void getTableMetadata() { @Test void getTableMetadataWithTimestamp() { var PTable = new SharedTable( - "icebergtable2", - "default", - "share1", - icebergTableWithHadoopCatalog("test_db", "icebergtable2")); + "icebergtable2", + "default", + "share1", + icebergTableWithHadoopCatalog("test_db", "icebergtable2")); var DTable = icebergTableLoader.loadTable(PTable); var metadata = DTable.getMetadata(Optional.of("2024-01-25T01:32:00+01:00")); assertTrue(metadata.isPresent()); @@ -53,10 +53,10 @@ void getUnknownTableMetadata() { @Test void getTableVersion() { var PTable = new SharedTable( - "icebergtable1", - "default", - "share1", - icebergTableWithHadoopCatalog("test_db", "icebergtable1")); + "icebergtable1", + "default", + "share1", + icebergTableWithHadoopCatalog("test_db", "icebergtable1")); var DTable = icebergTableLoader.loadTable(PTable); var version = DTable.getTableVersion(Optional.empty()); assertTrue(version.isPresent()); @@ -66,10 +66,10 @@ void getTableVersion() { @Test void getTableVersionWithTimestamp() { var PTable = new SharedTable( - "icebergtable2", - "default", - "share1", - icebergTableWithHadoopCatalog("test_db", "icebergtable2")); + "icebergtable2", + "default", + "share1", + icebergTableWithHadoopCatalog("test_db", "icebergtable2")); var DTable = icebergTableLoader.loadTable(PTable); var version = DTable.getTableVersion(Optional.of("2024-01-25T01:32:00+01:00")); assertTrue(version.isPresent()); From 88b053f0c45267323ce3618efc901e4faa91a742 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Thu, 25 Jan 2024 02:34:05 +0100 Subject: [PATCH 3/7] added an iceberg table wiht history to tests --- .../test/java/io/whitefox/api/deltasharing/SampleTables.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java index a7a069505..a3a57208f 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java @@ -62,8 +62,7 @@ public static StorageManager createStorageManager() { new SharedTable("table1", "default", "name", deltaTable1), new SharedTable( "table-with-history", "default", "name", deltaTableWithHistory1), - new SharedTable("icebergtable1", "default", "name", icebergtable1), - new SharedTable("icebergtable2", "default", "name", icebergtable2)), + new SharedTable("icebergtable1", "default", "name", icebergtable1)), "name")), testPrincipal, 0L))); From c5ca83e6599a74febc06f84b6cce7f25a15875f0 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Thu, 25 Jan 2024 03:01:31 +0100 Subject: [PATCH 4/7] added an iceberg table wiht history to tests --- .../test/java/io/whitefox/api/deltasharing/SampleTables.java | 3 ++- .../api/deltasharing/server/DeltaSharesApiImplTest.java | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java index a3a57208f..a7a069505 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java @@ -62,7 +62,8 @@ public static StorageManager createStorageManager() { new SharedTable("table1", "default", "name", deltaTable1), new SharedTable( "table-with-history", "default", "name", deltaTableWithHistory1), - new SharedTable("icebergtable1", "default", "name", icebergtable1)), + new SharedTable("icebergtable1", "default", "name", icebergtable1), + new SharedTable("icebergtable2", "default", "name", icebergtable2)), "name")), testPrincipal, 0L))); diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java index 389da74bf..a8dd7ed2b 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java @@ -149,10 +149,10 @@ public void listTables() { .get("delta-api/v1/shares/{share}/schemas/{schema}/tables", "name", "default") .then() .statusCode(200) - .body("items", hasSize(3)) + .body("items", hasSize(4)) .body( "items[0].name", - either(is("table1")).or(is("table-with-history")).or(is("icebergtable1"))) + either(is("table1")).or(is("table-with-history")).or(is("icebergtable1")).or(is("icebergtable2"))) .body("items[0].schema", is("default")) .body("items[0].share", is("name")) .body("nextPageToken", is(nullValue())); From 5dd500c3b3215505c8d4bf4332df74e8eaa1a421 Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Thu, 25 Jan 2024 03:09:26 +0100 Subject: [PATCH 5/7] added history table to tests - finally --- .../deltasharing/server/DeltaSharesApiImplTest.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java index a8dd7ed2b..81d667a7d 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java @@ -152,7 +152,10 @@ public void listTables() { .body("items", hasSize(4)) .body( "items[0].name", - either(is("table1")).or(is("table-with-history")).or(is("icebergtable1")).or(is("icebergtable2"))) + either(is("table1")) + .or(is("table-with-history")) + .or(is("icebergtable1")) + .or(is("icebergtable2"))) .body("items[0].schema", is("default")) .body("items[0].share", is("name")) .body("nextPageToken", is(nullValue())); @@ -248,10 +251,13 @@ public void listAllTables() { .get("delta-api/v1/shares/{share}/all-tables", "name") .then() .statusCode(200) - .body("items", hasSize(3)) + .body("items", hasSize(4)) .body( "items[0].name", - either(is("table1")).or(is("table-with-history")).or(is("icebergtable1"))) + either(is("table1")) + .or(is("table-with-history")) + .or(is("icebergtable1")) + .or(is("icebergtable2"))) .body("items[0].schema", is("default")) .body("items[0].share", is("name")) .body("nextPageToken", is(nullValue())); From d5b6bf14d7398a80f7f7ee4569229c08bc84333d Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 26 Jan 2024 14:31:10 +0100 Subject: [PATCH 6/7] use iceberg api to find a timestamp id --- .../core/services/IcebergSharedTable.java | 21 +++++++++++-------- .../core/services/IcebergSharedTableTest.java | 16 ++++++++++++-- 2 files changed, 26 insertions(+), 11 deletions(-) diff --git a/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java index f55e2dafe..db43cc43d 100644 --- a/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java @@ -7,15 +7,13 @@ import java.sql.Timestamp; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; -import java.util.Comparator; import java.util.Optional; import java.util.stream.Collectors; -import java.util.stream.Stream; -import java.util.stream.StreamSupport; import org.apache.commons.lang3.NotImplementedException; import org.apache.iceberg.PartitionField; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.util.SnapshotUtil; public class IcebergSharedTable implements InternalSharedTable { @@ -62,12 +60,17 @@ private Optional getSnapshot(Optional startingTimestamp) { return startingTimestamp .map(this::getTimestamp) .map(Timestamp::getTime) - .map(t -> StreamSupport.stream(icebergTable.snapshots().spliterator(), false) - .sorted(Comparator.comparingLong(Snapshot::timestampMillis)) - .filter(s -> s.timestampMillis() > t)) - .map(Stream::findFirst) - .map(s -> s.or(Optional::empty)) - .orElseGet(() -> Optional.of(icebergTable.currentSnapshot())); + .map(this::getSnapshotForTimestampAsOf) + .orElseGet(() -> Optional.ofNullable(icebergTable.currentSnapshot())); + } + + private Optional getSnapshotForTimestampAsOf(long l) { + try { + return Optional.of(SnapshotUtil.snapshotIdAsOfTime(icebergTable, l)) + .map(icebergTable::snapshot); + } catch (IllegalArgumentException iea) { + return Optional.empty(); + } } private Timestamp getTimestamp(String timestamp) { diff --git a/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java index 660a57888..0c25ac221 100644 --- a/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java @@ -38,7 +38,7 @@ void getTableMetadataWithTimestamp() { "share1", icebergTableWithHadoopCatalog("test_db", "icebergtable2")); var DTable = icebergTableLoader.loadTable(PTable); - var metadata = DTable.getMetadata(Optional.of("2024-01-25T01:32:00+01:00")); + var metadata = DTable.getMetadata(Optional.of("2024-01-25T01:32:15+01:00")); assertTrue(metadata.isPresent()); assertEquals("2174306913745765008", metadata.get().id()); } @@ -71,8 +71,20 @@ void getTableVersionWithTimestamp() { "share1", icebergTableWithHadoopCatalog("test_db", "icebergtable2")); var DTable = icebergTableLoader.loadTable(PTable); - var version = DTable.getTableVersion(Optional.of("2024-01-25T01:32:00+01:00")); + var version = DTable.getTableVersion(Optional.of("2024-01-25T01:32:15+01:00")); assertTrue(version.isPresent()); assertEquals(1, version.get()); } + + @Test + void getTableVersionWithTooOldTimestamp() { + var PTable = new SharedTable( + "icebergtable2", + "default", + "share1", + icebergTableWithHadoopCatalog("test_db", "icebergtable2")); + var DTable = icebergTableLoader.loadTable(PTable); + var version = DTable.getTableVersion(Optional.of("2024-01-24T01:32:15+01:00")); + assertTrue(version.isEmpty()); + } } From 27c785c36e8191ac5f567d5d7d5c7ecc848212ce Mon Sep 17 00:00:00 2001 From: Aleksandar Milosevic Date: Fri, 26 Jan 2024 17:29:25 +0100 Subject: [PATCH 7/7] added api tests --- .../server/DeltaSharesApiImplAwsTest.java | 16 ++++++++++++++++ .../server/DeltaSharesApiImplTest.java | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplAwsTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplAwsTest.java index 22d11d591..aa84edf03 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplAwsTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplAwsTest.java @@ -88,6 +88,22 @@ public void updateStorageManagerWithS3Tables() { 0L)); } + @Test + @DisabledOnOs(OS.WINDOWS) + public void icebergTableVersion() { + given() + .when() + .filter(deltaFilter) + .get( + "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/version", + "s3share", + "s3schema", + "s3IcebergTable1") + .then() + .statusCode(200) + .header("Delta-Table-Version", "1"); + } + @Test @DisabledOnOs(OS.WINDOWS) public void icebergTableMetadata() throws IOException { diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java index 81d667a7d..d1c0119bd 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java @@ -209,6 +209,22 @@ public void deltaTableMetadata() throws IOException { objectMapper.reader().readValue(responseBodyLines[1], MetadataObject.class)); } + @Test + @DisabledOnOs(OS.WINDOWS) + public void icebergTableVersion() { + given() + .when() + .filter(deltaFilter) + .get( + "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/version", + "name", + "default", + "icebergtable1") + .then() + .statusCode(200) + .header("Delta-Table-Version", "1"); + } + @Test @DisabledOnOs(OS.WINDOWS) public void icebergTableMetadata() throws IOException {