diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java index cd18041e3..a7a069505 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/SampleTables.java @@ -41,6 +41,9 @@ public static InternalTable s3IcebergTable1( public static final InternalTable icebergtable1 = icebergTableWithHadoopCatalog("test_db", "icebergtable1"); + public static final InternalTable icebergtable2 = + icebergTableWithHadoopCatalog("test_db", "icebergtable2"); + public static final String deltaTable1Path = deltaTableUri("delta-table"); public static final String deltaTableWithHistory1Path = deltaTableUri("delta-table-with-history"); @@ -59,7 +62,8 @@ public static StorageManager createStorageManager() { new SharedTable("table1", "default", "name", deltaTable1), new SharedTable( "table-with-history", "default", "name", deltaTableWithHistory1), - new SharedTable("icebergtable1", "default", "name", icebergtable1)), + new SharedTable("icebergtable1", "default", "name", icebergtable1), + new SharedTable("icebergtable2", "default", "name", icebergtable2)), "name")), testPrincipal, 0L))); diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplAwsTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplAwsTest.java index 22d11d591..aa84edf03 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplAwsTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplAwsTest.java @@ -88,6 +88,22 @@ public void updateStorageManagerWithS3Tables() { 0L)); } + @Test + @DisabledOnOs(OS.WINDOWS) + public void icebergTableVersion() { + given() + .when() + .filter(deltaFilter) + .get( + "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/version", + "s3share", + "s3schema", + "s3IcebergTable1") + .then() + .statusCode(200) + .header("Delta-Table-Version", "1"); + } + @Test @DisabledOnOs(OS.WINDOWS) public void icebergTableMetadata() throws IOException { diff --git a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java index 389da74bf..d1c0119bd 100644 --- a/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java +++ b/server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java @@ -149,10 +149,13 @@ public void listTables() { .get("delta-api/v1/shares/{share}/schemas/{schema}/tables", "name", "default") .then() .statusCode(200) - .body("items", hasSize(3)) + .body("items", hasSize(4)) .body( "items[0].name", - either(is("table1")).or(is("table-with-history")).or(is("icebergtable1"))) + either(is("table1")) + .or(is("table-with-history")) + .or(is("icebergtable1")) + .or(is("icebergtable2"))) .body("items[0].schema", is("default")) .body("items[0].share", is("name")) .body("nextPageToken", is(nullValue())); @@ -206,6 +209,22 @@ public void deltaTableMetadata() throws IOException { objectMapper.reader().readValue(responseBodyLines[1], MetadataObject.class)); } + @Test + @DisabledOnOs(OS.WINDOWS) + public void icebergTableVersion() { + given() + .when() + .filter(deltaFilter) + .get( + "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/version", + "name", + "default", + "icebergtable1") + .then() + .statusCode(200) + .header("Delta-Table-Version", "1"); + } + @Test @DisabledOnOs(OS.WINDOWS) public void icebergTableMetadata() throws IOException { @@ -248,10 +267,13 @@ public void listAllTables() { .get("delta-api/v1/shares/{share}/all-tables", "name") .then() .statusCode(200) - .body("items", hasSize(3)) + .body("items", hasSize(4)) .body( "items[0].name", - either(is("table1")).or(is("table-with-history")).or(is("icebergtable1"))) + either(is("table1")) + .or(is("table-with-history")) + .or(is("icebergtable1")) + .or(is("icebergtable2"))) .body("items[0].schema", is("default")) .body("items[0].share", is("name")) .body("nextPageToken", is(nullValue())); diff --git a/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java b/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java index ac6225762..db43cc43d 100644 --- a/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java +++ b/server/core/src/main/java/io/whitefox/core/services/IcebergSharedTable.java @@ -13,6 +13,7 @@ import org.apache.iceberg.PartitionField; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.util.SnapshotUtil; public class IcebergSharedTable implements InternalSharedTable { @@ -59,8 +60,17 @@ private Optional getSnapshot(Optional startingTimestamp) { return startingTimestamp .map(this::getTimestamp) .map(Timestamp::getTime) - .map(icebergTable::snapshot) - .or(() -> Optional.of(icebergTable.currentSnapshot())); + .map(this::getSnapshotForTimestampAsOf) + .orElseGet(() -> Optional.ofNullable(icebergTable.currentSnapshot())); + } + + private Optional getSnapshotForTimestampAsOf(long l) { + try { + return Optional.of(SnapshotUtil.snapshotIdAsOfTime(icebergTable, l)) + .map(icebergTable::snapshot); + } catch (IllegalArgumentException iea) { + return Optional.empty(); + } } private Timestamp getTimestamp(String timestamp) { @@ -71,7 +81,7 @@ private Timestamp getTimestamp(String timestamp) { @Override public Optional getTableVersion(Optional startingTimestamp) { - return Optional.of(0L); + return getSnapshot(startingTimestamp).map(Snapshot::sequenceNumber); } @Override diff --git a/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java b/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java index 2fdd76a29..0c25ac221 100644 --- a/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java +++ b/server/core/src/test/java/io/whitefox/core/services/IcebergSharedTableTest.java @@ -30,10 +30,61 @@ void getTableMetadata() { assertEquals("3369848726892806393", metadata.get().id()); } + @Test + void getTableMetadataWithTimestamp() { + var PTable = new SharedTable( + "icebergtable2", + "default", + "share1", + icebergTableWithHadoopCatalog("test_db", "icebergtable2")); + var DTable = icebergTableLoader.loadTable(PTable); + var metadata = DTable.getMetadata(Optional.of("2024-01-25T01:32:15+01:00")); + assertTrue(metadata.isPresent()); + assertEquals("2174306913745765008", metadata.get().id()); + } + @Test void getUnknownTableMetadata() { var unknownPTable = new SharedTable( "notFound", "default", "share1", icebergTableWithHadoopCatalog("test_db", "not-found")); assertThrows(IllegalArgumentException.class, () -> DeltaSharedTable.of(unknownPTable)); } + + @Test + void getTableVersion() { + var PTable = new SharedTable( + "icebergtable1", + "default", + "share1", + icebergTableWithHadoopCatalog("test_db", "icebergtable1")); + var DTable = icebergTableLoader.loadTable(PTable); + var version = DTable.getTableVersion(Optional.empty()); + assertTrue(version.isPresent()); + assertEquals(1, version.get()); + } + + @Test + void getTableVersionWithTimestamp() { + var PTable = new SharedTable( + "icebergtable2", + "default", + "share1", + icebergTableWithHadoopCatalog("test_db", "icebergtable2")); + var DTable = icebergTableLoader.loadTable(PTable); + var version = DTable.getTableVersion(Optional.of("2024-01-25T01:32:15+01:00")); + assertTrue(version.isPresent()); + assertEquals(1, version.get()); + } + + @Test + void getTableVersionWithTooOldTimestamp() { + var PTable = new SharedTable( + "icebergtable2", + "default", + "share1", + icebergTableWithHadoopCatalog("test_db", "icebergtable2")); + var DTable = icebergTableLoader.loadTable(PTable); + var version = DTable.getTableVersion(Optional.of("2024-01-24T01:32:15+01:00")); + assertTrue(version.isEmpty()); + } } diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00001-1-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00001-1-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc new file mode 100644 index 000000000..88849bd70 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00001-1-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00001-9-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00001-9-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc new file mode 100644 index 000000000..c2a62a9e7 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00001-9-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00003-11-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00003-11-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc new file mode 100644 index 000000000..36b4662e7 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00003-11-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00003-3-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00003-3-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc new file mode 100644 index 000000000..cae6372c5 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00003-3-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00004-12-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00004-12-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc new file mode 100644 index 000000000..0c4d58486 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00004-12-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00004-4-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00004-4-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc new file mode 100644 index 000000000..0e006ee21 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00004-4-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00006-14-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00006-14-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc new file mode 100644 index 000000000..5060ac6bc Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00006-14-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00006-6-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00006-6-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc new file mode 100644 index 000000000..eed31addd Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00006-6-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00007-15-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00007-15-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc new file mode 100644 index 000000000..a1fce5513 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00007-15-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00007-7-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00007-7-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc new file mode 100644 index 000000000..b72dd1703 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/.00007-7-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00001-1-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00001-1-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet new file mode 100644 index 000000000..211d5f907 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00001-1-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00001-9-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00001-9-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet new file mode 100644 index 000000000..60840c57b Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00001-9-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00003-11-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00003-11-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet new file mode 100644 index 000000000..86ffe8162 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00003-11-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00003-3-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00003-3-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet new file mode 100644 index 000000000..9ab441824 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00003-3-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00004-12-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00004-12-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet new file mode 100644 index 000000000..68b37ff08 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00004-12-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00004-4-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00004-4-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet new file mode 100644 index 000000000..00a304e2d Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00004-4-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00006-14-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00006-14-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet new file mode 100644 index 000000000..78377a15d Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00006-14-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00006-6-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00006-6-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet new file mode 100644 index 000000000..8b0078ec9 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00006-6-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00007-15-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00007-15-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet new file mode 100644 index 000000000..7d3f007f7 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00007-15-eb480a8c-25d9-444b-917d-d77c5cfe5642-00001.parquet differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00007-7-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00007-7-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet new file mode 100644 index 000000000..6377a5954 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/data/00007-7-b481540c-b583-4d82-9097-2fd746120aae-00001.parquet differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.ac1c3eb8-a744-47c9-8f48-535f147b5df1-m0.avro.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.ac1c3eb8-a744-47c9-8f48-535f147b5df1-m0.avro.crc new file mode 100644 index 000000000..73a65f1f7 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.ac1c3eb8-a744-47c9-8f48-535f147b5df1-m0.avro.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.f034929c-0fad-4fed-9671-579ecceb195b-m0.avro.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.f034929c-0fad-4fed-9671-579ecceb195b-m0.avro.crc new file mode 100644 index 000000000..f1f1d1e1d Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.f034929c-0fad-4fed-9671-579ecceb195b-m0.avro.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro.crc new file mode 100644 index 000000000..d74c496a0 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro.crc new file mode 100644 index 000000000..d60fe28ce Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v1.metadata.json.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v1.metadata.json.crc new file mode 100644 index 000000000..3ff5d1018 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v1.metadata.json.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v2.metadata.json.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v2.metadata.json.crc new file mode 100644 index 000000000..0b5ac8526 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v2.metadata.json.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v3.metadata.json.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v3.metadata.json.crc new file mode 100644 index 000000000..5c44f6ebb Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.v3.metadata.json.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.version-hint.text.crc b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.version-hint.text.crc new file mode 100644 index 000000000..65f981923 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/.version-hint.text.crc differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/ac1c3eb8-a744-47c9-8f48-535f147b5df1-m0.avro b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/ac1c3eb8-a744-47c9-8f48-535f147b5df1-m0.avro new file mode 100644 index 000000000..18afad317 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/ac1c3eb8-a744-47c9-8f48-535f147b5df1-m0.avro differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/f034929c-0fad-4fed-9671-579ecceb195b-m0.avro b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/f034929c-0fad-4fed-9671-579ecceb195b-m0.avro new file mode 100644 index 000000000..45d9aaab4 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/f034929c-0fad-4fed-9671-579ecceb195b-m0.avro differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro new file mode 100644 index 000000000..31e7e930b Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro new file mode 100644 index 000000000..a1b42d070 Binary files /dev/null and b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro differ diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v1.metadata.json b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v1.metadata.json new file mode 100644 index 000000000..3c1770f16 --- /dev/null +++ b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v1.metadata.json @@ -0,0 +1,39 @@ +{ + "format-version" : 2, + "table-uuid" : "77734b3f-6c0a-4151-83b9-9be5cf0197d9", + "location" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2", + "last-sequence-number" : 0, + "last-updated-ms" : 1706142410682, + "last-column-id" : 1, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : true, + "type" : "long" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : -1, + "refs" : { }, + "snapshots" : [ ], + "statistics" : [ ], + "snapshot-log" : [ ], + "metadata-log" : [ ] +} \ No newline at end of file diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v2.metadata.json b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v2.metadata.json new file mode 100644 index 000000000..6d42ec518 --- /dev/null +++ b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v2.metadata.json @@ -0,0 +1,70 @@ +{ + "format-version" : 2, + "table-uuid" : "77734b3f-6c0a-4151-83b9-9be5cf0197d9", + "location" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2", + "last-sequence-number" : 1, + "last-updated-ms" : 1706142720036, + "last-column-id" : 1, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : true, + "type" : "long" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 2174306913745765008, + "refs" : { + "main" : { + "snapshot-id" : 2174306913745765008, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 2174306913745765008, + "timestamp-ms" : 1706142720036, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1706142687377", + "added-data-files" : "5", + "added-records" : "5", + "added-files-size" : "2094", + "changed-partition-count" : "1", + "total-records" : "5", + "total-files-size" : "2094", + "total-data-files" : "5", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1706142720036, + "snapshot-id" : 2174306913745765008 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1706142410682, + "metadata-file" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v1.metadata.json" + } ] +} \ No newline at end of file diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v3.metadata.json b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v3.metadata.json new file mode 100644 index 000000000..30dcb3b4c --- /dev/null +++ b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v3.metadata.json @@ -0,0 +1,97 @@ +{ + "format-version" : 2, + "table-uuid" : "77734b3f-6c0a-4151-83b9-9be5cf0197d9", + "location" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2", + "last-sequence-number" : 2, + "last-updated-ms" : 1706142746160, + "last-column-id" : 1, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "id", + "required" : true, + "type" : "long" + } ] + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ ] + } ], + "last-partition-id" : 999, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "write.parquet.compression-codec" : "zstd" + }, + "current-snapshot-id" : 3375739370786085828, + "refs" : { + "main" : { + "snapshot-id" : 3375739370786085828, + "type" : "branch" + } + }, + "snapshots" : [ { + "sequence-number" : 1, + "snapshot-id" : 2174306913745765008, + "timestamp-ms" : 1706142720036, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1706142687377", + "added-data-files" : "5", + "added-records" : "5", + "added-files-size" : "2094", + "changed-partition-count" : "1", + "total-records" : "5", + "total-files-size" : "2094", + "total-data-files" : "5", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-2174306913745765008-1-f034929c-0fad-4fed-9671-579ecceb195b.avro", + "schema-id" : 0 + }, { + "sequence-number" : 2, + "snapshot-id" : 3375739370786085828, + "parent-snapshot-id" : 2174306913745765008, + "timestamp-ms" : 1706142746160, + "summary" : { + "operation" : "append", + "spark.app.id" : "local-1706142687377", + "added-data-files" : "5", + "added-records" : "5", + "added-files-size" : "2095", + "changed-partition-count" : "1", + "total-records" : "10", + "total-files-size" : "4189", + "total-data-files" : "10", + "total-delete-files" : "0", + "total-position-deletes" : "0", + "total-equality-deletes" : "0" + }, + "manifest-list" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/snap-3375739370786085828-1-ac1c3eb8-a744-47c9-8f48-535f147b5df1.avro", + "schema-id" : 0 + } ], + "statistics" : [ ], + "snapshot-log" : [ { + "timestamp-ms" : 1706142720036, + "snapshot-id" : 2174306913745765008 + }, { + "timestamp-ms" : 1706142746160, + "snapshot-id" : 3375739370786085828 + } ], + "metadata-log" : [ { + "timestamp-ms" : 1706142410682, + "metadata-file" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v1.metadata.json" + }, { + "timestamp-ms" : 1706142720036, + "metadata-file" : "/Users/aleksandarmilosevic/Desktop/lake-sharing/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/v2.metadata.json" + } ] +} \ No newline at end of file diff --git a/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/version-hint.text b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/version-hint.text new file mode 100644 index 000000000..e440e5c84 --- /dev/null +++ b/server/core/src/testFixtures/resources/iceberg/samples/test_db/icebergtable2/metadata/version-hint.text @@ -0,0 +1 @@ +3 \ No newline at end of file