diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml index 21abbeb19ab7..2c5c0c69ae49 100644 --- a/.palantir/revapi.yml +++ b/.palantir/revapi.yml @@ -801,6 +801,14 @@ acceptedBreaks: - code: "java.method.removed" old: "method org.apache.iceberg.view.ViewBuilder org.apache.iceberg.view.ViewBuilder::withQueryColumnNames(java.util.List)" justification: "Acceptable break due to updating View APIs and the View Spec" + org.apache.iceberg:iceberg-core: + - code: "java.method.abstractMethodAdded" + new: "method org.apache.iceberg.io.FileAppender>\ + \ org.apache.iceberg.ManifestWriter>>::newAppender(org.apache.iceberg.PartitionSpec,\ + \ org.apache.iceberg.io.OutputFile, java.util.Map)" + justification: "{Adding a new method to support set file appender config for\ + \ ManifestWriter}" apache-iceberg-0.14.0: org.apache.iceberg:iceberg-api: - code: "java.class.defaultSerializationChanged" diff --git a/build.gradle b/build.gradle index 0f5e15cde4b4..d5af6231b9f6 100644 --- a/build.gradle +++ b/build.gradle @@ -360,6 +360,8 @@ project(':iceberg-core') { testImplementation libs.esotericsoftware.kryo testImplementation libs.guava.testlib testImplementation libs.awaitility + testImplementation libs.snappy + testImplementation libs.zstd } } diff --git a/core/src/jmh/java/org/apache/iceberg/ManifestReadBenchmark.java b/core/src/jmh/java/org/apache/iceberg/ManifestReadBenchmark.java index 6677e5d8b651..c536d25990b0 100644 --- a/core/src/jmh/java/org/apache/iceberg/ManifestReadBenchmark.java +++ b/core/src/jmh/java/org/apache/iceberg/ManifestReadBenchmark.java @@ -18,6 +18,8 @@ */ package org.apache.iceberg; +import static org.apache.iceberg.TableProperties.AVRO_COMPRESSION; + import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; @@ -41,6 +43,7 @@ import org.openjdk.jmh.annotations.Fork; import org.openjdk.jmh.annotations.Measurement; import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Param; import org.openjdk.jmh.annotations.Scope; import org.openjdk.jmh.annotations.Setup; import org.openjdk.jmh.annotations.State; @@ -59,14 +62,20 @@ public class ManifestReadBenchmark { private static final int NUM_ROWS = 100000; private static final int NUM_COLS = 10; - private String baseDir; + private String rootPath; private String manifestListFile; + @Param({"UNCOMPRESSED", "SNAPPY", "ZSTD", "GZIP"}) + private String codec; + @Setup public void before() { - baseDir = + String tmpDir = Paths.get(new File(System.getProperty("java.io.tmpdir")).getAbsolutePath()).toString(); - manifestListFile = String.format("%s/%s.avro", baseDir, UUID.randomUUID()); + File baseDirectory = new File(tmpDir, codec); + baseDirectory.mkdir(); + rootPath = baseDirectory.getAbsolutePath(); + manifestListFile = String.format("%s/%s.avro", rootPath, UUID.randomUUID()); Random random = new Random(System.currentTimeMillis()); ManifestListWriter listWriter = @@ -76,10 +85,15 @@ public void before() { for (int i = 0; i < NUM_FILES; i++) { OutputFile manifestFile = org.apache.iceberg.Files.localOutput( - String.format("%s/%s.avro", baseDir, UUID.randomUUID())); + String.format("%s/%s.avro", rootPath, UUID.randomUUID())); ManifestWriter writer = - ManifestFiles.write(1, PartitionSpec.unpartitioned(), manifestFile, 1L); + ManifestFiles.write( + 1, + PartitionSpec.unpartitioned(), + manifestFile, + 1L, + ImmutableMap.of(AVRO_COMPRESSION, codec)); try (ManifestWriter finalWriter = writer) { for (int j = 0; j < NUM_ROWS; j++) { DataFile dataFile = @@ -107,11 +121,11 @@ public void before() { @TearDown public void after() throws IOException { - if (baseDir != null) { - try (Stream walk = Files.walk(Paths.get(baseDir))) { + if (rootPath != null) { + try (Stream walk = Files.walk(Paths.get(rootPath))) { walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); } - baseDir = null; + rootPath = null; } manifestListFile = null; diff --git a/core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java b/core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java index 54bf3c6e44c4..6ab6ad92e7ef 100644 --- a/core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java +++ b/core/src/main/java/org/apache/iceberg/BaseRewriteManifests.java @@ -170,7 +170,8 @@ private ManifestFile copyManifest(ManifestFile manifest) { specsById, newFile, snapshotId(), - summaryBuilder); + summaryBuilder, + ops.current().properties()); } @Override diff --git a/core/src/main/java/org/apache/iceberg/FastAppend.java b/core/src/main/java/org/apache/iceberg/FastAppend.java index 3079757392cd..285d606ba0e5 100644 --- a/core/src/main/java/org/apache/iceberg/FastAppend.java +++ b/core/src/main/java/org/apache/iceberg/FastAppend.java @@ -138,7 +138,8 @@ private ManifestFile copyManifest(ManifestFile manifest) { current.specsById(), newManifestPath, snapshotId(), - summaryBuilder); + summaryBuilder, + current.properties()); } @Override diff --git a/core/src/main/java/org/apache/iceberg/ManifestFiles.java b/core/src/main/java/org/apache/iceberg/ManifestFiles.java index c23ab667a41b..801a74c392b9 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestFiles.java +++ b/core/src/main/java/org/apache/iceberg/ManifestFiles.java @@ -157,11 +157,30 @@ public static ManifestWriter write(PartitionSpec spec, OutputFile outp */ public static ManifestWriter write( int formatVersion, PartitionSpec spec, OutputFile outputFile, Long snapshotId) { + return write(formatVersion, spec, outputFile, snapshotId, ImmutableMap.of()); + } + + /** + * Create a new {@link ManifestWriter} for the given format version. + * + * @param formatVersion a target format version + * @param spec a {@link PartitionSpec} + * @param outputFile an {@link OutputFile} where the manifest will be written + * @param snapshotId a snapshot ID for the manifest entries, or null for an inherited ID + * @param config config for manifest writer + * @return a manifest writer + */ + public static ManifestWriter write( + int formatVersion, + PartitionSpec spec, + OutputFile outputFile, + Long snapshotId, + Map config) { switch (formatVersion) { case 1: - return new ManifestWriter.V1Writer(spec, outputFile, snapshotId); + return new ManifestWriter.V1Writer(spec, outputFile, snapshotId, config); case 2: - return new ManifestWriter.V2Writer(spec, outputFile, snapshotId); + return new ManifestWriter.V2Writer(spec, outputFile, snapshotId, config); } throw new UnsupportedOperationException( "Cannot write manifest for table version: " + formatVersion); @@ -198,11 +217,30 @@ public static ManifestReader readDeleteManifest( */ public static ManifestWriter writeDeleteManifest( int formatVersion, PartitionSpec spec, OutputFile outputFile, Long snapshotId) { + return writeDeleteManifest(formatVersion, spec, outputFile, snapshotId, ImmutableMap.of()); + } + + /** + * Create a new {@link ManifestWriter} for the given format version. + * + * @param formatVersion a target format version + * @param spec a {@link PartitionSpec} + * @param outputFile an {@link OutputFile} where the manifest will be written + * @param snapshotId a snapshot ID for the manifest entries, or null for an inherited ID + * @param config config for manifest writer + * @return a manifest writer + */ + public static ManifestWriter writeDeleteManifest( + int formatVersion, + PartitionSpec spec, + OutputFile outputFile, + Long snapshotId, + Map config) { switch (formatVersion) { case 1: throw new IllegalArgumentException("Cannot write delete files in a v1 table"); case 2: - return new ManifestWriter.V2DeleteWriter(spec, outputFile, snapshotId); + return new ManifestWriter.V2DeleteWriter(spec, outputFile, snapshotId, config); } throw new UnsupportedOperationException( "Cannot write manifest for table version: " + formatVersion); @@ -256,7 +294,8 @@ static ManifestFile copyAppendManifest( Map specsById, OutputFile outputFile, long snapshotId, - SnapshotSummary.Builder summaryBuilder) { + SnapshotSummary.Builder summaryBuilder, + Map config) { // use metadata that will add the current snapshot's ID for the rewrite InheritableMetadata inheritableMetadata = InheritableMetadataFactory.forCopy(snapshotId); try (ManifestReader reader = @@ -267,7 +306,8 @@ static ManifestFile copyAppendManifest( outputFile, snapshotId, summaryBuilder, - ManifestEntry.Status.ADDED); + ManifestEntry.Status.ADDED, + config); } catch (IOException e) { throw new RuntimeIOException(e, "Failed to close manifest: %s", toCopy.location()); } @@ -280,7 +320,8 @@ static ManifestFile copyRewriteManifest( Map specsById, OutputFile outputFile, long snapshotId, - SnapshotSummary.Builder summaryBuilder) { + SnapshotSummary.Builder summaryBuilder, + Map config) { // for a rewritten manifest all snapshot ids should be set. use empty metadata to throw an // exception if it is not InheritableMetadata inheritableMetadata = InheritableMetadataFactory.empty(); @@ -292,7 +333,8 @@ static ManifestFile copyRewriteManifest( outputFile, snapshotId, summaryBuilder, - ManifestEntry.Status.EXISTING); + ManifestEntry.Status.EXISTING, + config); } catch (IOException e) { throw new RuntimeIOException(e, "Failed to close manifest: %s", toCopy.location()); } @@ -305,8 +347,10 @@ private static ManifestFile copyManifestInternal( OutputFile outputFile, long snapshotId, SnapshotSummary.Builder summaryBuilder, - ManifestEntry.Status allowedEntryStatus) { - ManifestWriter writer = write(formatVersion, reader.spec(), outputFile, snapshotId); + ManifestEntry.Status allowedEntryStatus, + Map config) { + ManifestWriter writer = + write(formatVersion, reader.spec(), outputFile, snapshotId, config); boolean threw = true; try { for (ManifestEntry entry : reader.entries()) { diff --git a/core/src/main/java/org/apache/iceberg/ManifestWriter.java b/core/src/main/java/org/apache/iceberg/ManifestWriter.java index 4865ccfc3b2d..a6fdffa244fb 100644 --- a/core/src/main/java/org/apache/iceberg/ManifestWriter.java +++ b/core/src/main/java/org/apache/iceberg/ManifestWriter.java @@ -19,11 +19,13 @@ package org.apache.iceberg; import java.io.IOException; +import java.util.Map; import org.apache.iceberg.avro.Avro; import org.apache.iceberg.exceptions.RuntimeIOException; import org.apache.iceberg.io.FileAppender; import org.apache.iceberg.io.OutputFile; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; /** * Writer for manifest files. @@ -38,6 +40,8 @@ public abstract class ManifestWriter> implements FileAp private final OutputFile file; private final int specId; + // Config for file appender, such as for compression codec + private final Map config; private final FileAppender> writer; private final Long snapshotId; private final GenericManifestEntry reused; @@ -52,10 +56,12 @@ public abstract class ManifestWriter> implements FileAp private long deletedRows = 0L; private Long minDataSequenceNumber = null; - private ManifestWriter(PartitionSpec spec, OutputFile file, Long snapshotId) { + private ManifestWriter( + PartitionSpec spec, OutputFile file, Long snapshotId, Map config) { this.file = file; this.specId = spec.specId(); - this.writer = newAppender(spec, file); + this.config = config; + this.writer = newAppender(spec, file, config); this.snapshotId = snapshotId; this.reused = new GenericManifestEntry<>(spec.partitionType()); this.stats = new PartitionSummary(spec); @@ -63,9 +69,14 @@ private ManifestWriter(PartitionSpec spec, OutputFile file, Long snapshotId) { protected abstract ManifestEntry prepare(ManifestEntry entry); + /** @deprecated since 1.4.0, will be removed in 1.5.0 */ + @Deprecated protected abstract FileAppender> newAppender( PartitionSpec spec, OutputFile outputFile); + protected abstract FileAppender> newAppender( + PartitionSpec spec, OutputFile outputFile, Map configs); + protected ManifestContent content() { return ManifestContent.DATA; } @@ -216,8 +227,8 @@ public void close() throws IOException { static class V2Writer extends ManifestWriter { private final V2Metadata.IndexedManifestEntry entryWrapper; - V2Writer(PartitionSpec spec, OutputFile file, Long snapshotId) { - super(spec, file, snapshotId); + V2Writer(PartitionSpec spec, OutputFile file, Long snapshotId, Map config) { + super(spec, file, snapshotId, config); this.entryWrapper = new V2Metadata.IndexedManifestEntry<>(snapshotId, spec.partitionType()); } @@ -228,7 +239,13 @@ protected ManifestEntry prepare(ManifestEntry entry) { @Override protected FileAppender> newAppender( - PartitionSpec spec, OutputFile file) { + PartitionSpec spec, OutputFile outputFile) { + return newAppender(spec, outputFile, ImmutableMap.of()); + } + + @Override + protected FileAppender> newAppender( + PartitionSpec spec, OutputFile file, Map configs) { Schema manifestSchema = V2Metadata.entrySchema(spec.partitionType()); try { return Avro.write(file) @@ -239,6 +256,7 @@ protected FileAppender> newAppender( .meta("partition-spec-id", String.valueOf(spec.specId())) .meta("format-version", "2") .meta("content", "data") + .setAll(configs) .overwrite() .build(); } catch (IOException e) { @@ -250,8 +268,9 @@ protected FileAppender> newAppender( static class V2DeleteWriter extends ManifestWriter { private final V2Metadata.IndexedManifestEntry entryWrapper; - V2DeleteWriter(PartitionSpec spec, OutputFile file, Long snapshotId) { - super(spec, file, snapshotId); + V2DeleteWriter( + PartitionSpec spec, OutputFile file, Long snapshotId, Map config) { + super(spec, file, snapshotId, config); this.entryWrapper = new V2Metadata.IndexedManifestEntry<>(snapshotId, spec.partitionType()); } @@ -262,7 +281,13 @@ protected ManifestEntry prepare(ManifestEntry entry) { @Override protected FileAppender> newAppender( - PartitionSpec spec, OutputFile file) { + PartitionSpec spec, OutputFile outputFile) { + return newAppender(spec, outputFile, ImmutableMap.of()); + } + + @Override + protected FileAppender> newAppender( + PartitionSpec spec, OutputFile file, Map config) { Schema manifestSchema = V2Metadata.entrySchema(spec.partitionType()); try { return Avro.write(file) @@ -273,6 +298,7 @@ protected FileAppender> newAppender( .meta("partition-spec-id", String.valueOf(spec.specId())) .meta("format-version", "2") .meta("content", "deletes") + .setAll(config) .overwrite() .build(); } catch (IOException e) { @@ -289,8 +315,8 @@ protected ManifestContent content() { static class V1Writer extends ManifestWriter { private final V1Metadata.IndexedManifestEntry entryWrapper; - V1Writer(PartitionSpec spec, OutputFile file, Long snapshotId) { - super(spec, file, snapshotId); + V1Writer(PartitionSpec spec, OutputFile file, Long snapshotId, Map config) { + super(spec, file, snapshotId, config); this.entryWrapper = new V1Metadata.IndexedManifestEntry(spec.partitionType()); } @@ -301,7 +327,13 @@ protected ManifestEntry prepare(ManifestEntry entry) { @Override protected FileAppender> newAppender( - PartitionSpec spec, OutputFile file) { + PartitionSpec spec, OutputFile outputFile) { + return newAppender(spec, outputFile, ImmutableMap.of()); + } + + @Override + protected FileAppender> newAppender( + PartitionSpec spec, OutputFile file, Map config) { Schema manifestSchema = V1Metadata.entrySchema(spec.partitionType()); try { return Avro.write(file) @@ -311,6 +343,7 @@ protected FileAppender> newAppender( .meta("partition-spec", PartitionSpecParser.toJsonFields(spec)) .meta("partition-spec-id", String.valueOf(spec.specId())) .meta("format-version", "1") + .setAll(config) .overwrite() .build(); } catch (IOException e) { diff --git a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java index cb9a361ab283..cb1c9de02672 100644 --- a/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/MergingSnapshotProducer.java @@ -302,7 +302,8 @@ private ManifestFile copyManifest(ManifestFile manifest) { current.specsById(), newManifestPath, snapshotId(), - appendedManifestsSummary); + appendedManifestsSummary, + current.properties()); } /** diff --git a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java index 5a6a01ea062d..965901330289 100644 --- a/core/src/main/java/org/apache/iceberg/SnapshotProducer.java +++ b/core/src/main/java/org/apache/iceberg/SnapshotProducer.java @@ -492,12 +492,20 @@ protected OutputFile newManifestOutput() { protected ManifestWriter newManifestWriter(PartitionSpec spec) { return ManifestFiles.write( - ops.current().formatVersion(), spec, newManifestOutput(), snapshotId()); + ops.current().formatVersion(), + spec, + newManifestOutput(), + snapshotId(), + ops.current().properties()); } protected ManifestWriter newDeleteManifestWriter(PartitionSpec spec) { return ManifestFiles.writeDeleteManifest( - ops.current().formatVersion(), spec, newManifestOutput(), snapshotId()); + ops.current().formatVersion(), + spec, + newManifestOutput(), + snapshotId(), + ops.current().properties()); } protected RollingManifestWriter newRollingManifestWriter(PartitionSpec spec) { diff --git a/core/src/test/java/org/apache/iceberg/TableTestBase.java b/core/src/test/java/org/apache/iceberg/TableTestBase.java index a800214bc9a7..57b06415e570 100644 --- a/core/src/test/java/org/apache/iceberg/TableTestBase.java +++ b/core/src/test/java/org/apache/iceberg/TableTestBase.java @@ -28,6 +28,7 @@ import java.util.Arrays; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.UUID; import org.apache.iceberg.deletes.PositionDelete; @@ -254,16 +255,17 @@ ManifestFile writeManifest(Long snapshotId, DataFile... files) throws IOExceptio } ManifestFile writeManifest(String fileName, ManifestEntry... entries) throws IOException { - return writeManifest(null, fileName, entries); + return writeManifest(null, fileName, ImmutableMap.of(), entries); } ManifestFile writeManifest(Long snapshotId, ManifestEntry... entries) throws IOException { - return writeManifest(snapshotId, "input.m0.avro", entries); + return writeManifest(snapshotId, "input.m0.avro", ImmutableMap.of(), entries); } @SuppressWarnings("unchecked") > ManifestFile writeManifest( - Long snapshotId, String fileName, ManifestEntry... entries) throws IOException { + Long snapshotId, String fileName, Map config, ManifestEntry... entries) + throws IOException { File manifestFile = temp.newFile(fileName); Assert.assertTrue(manifestFile.delete()); OutputFile outputFile = table.ops().io().newOutputFile(manifestFile.getCanonicalPath()); @@ -272,12 +274,12 @@ > ManifestFile writeManifest( if (entries[0].file() instanceof DataFile) { writer = (ManifestWriter) - ManifestFiles.write(formatVersion, table.spec(), outputFile, snapshotId); + ManifestFiles.write(formatVersion, table.spec(), outputFile, snapshotId, config); } else { writer = (ManifestWriter) ManifestFiles.writeDeleteManifest( - formatVersion, table.spec(), outputFile, snapshotId); + formatVersion, table.spec(), outputFile, snapshotId, config); } try { for (ManifestEntry entry : entries) { diff --git a/core/src/test/java/org/apache/iceberg/TestManifestWriter.java b/core/src/test/java/org/apache/iceberg/TestManifestWriter.java index 17a41f418a8e..5b350c665737 100644 --- a/core/src/test/java/org/apache/iceberg/TestManifestWriter.java +++ b/core/src/test/java/org/apache/iceberg/TestManifestWriter.java @@ -18,13 +18,17 @@ */ package org.apache.iceberg; +import static org.apache.iceberg.TableProperties.AVRO_COMPRESSION; + import java.io.File; import java.io.IOException; import java.io.UncheckedIOException; import java.util.List; +import java.util.Map; import java.util.UUID; import org.apache.iceberg.ManifestEntry.Status; import org.apache.iceberg.io.OutputFile; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.apache.iceberg.types.Conversions; import org.apache.iceberg.types.Types; import org.assertj.core.api.Assertions; @@ -51,32 +55,7 @@ public TestManifestWriter(int formatVersion) { @Test public void testManifestStats() throws IOException { - ManifestFile manifest = - writeManifest( - "manifest.avro", - manifestEntry(Status.ADDED, null, newFile(10)), - manifestEntry(Status.ADDED, null, newFile(20)), - manifestEntry(Status.ADDED, null, newFile(5)), - manifestEntry(Status.ADDED, null, newFile(5)), - manifestEntry(Status.EXISTING, null, newFile(15)), - manifestEntry(Status.EXISTING, null, newFile(10)), - manifestEntry(Status.EXISTING, null, newFile(1)), - manifestEntry(Status.DELETED, null, newFile(5)), - manifestEntry(Status.DELETED, null, newFile(2))); - - Assert.assertTrue("Added files should be present", manifest.hasAddedFiles()); - Assert.assertEquals("Added files count should match", 4, (int) manifest.addedFilesCount()); - Assert.assertEquals("Added rows count should match", 40L, (long) manifest.addedRowsCount()); - - Assert.assertTrue("Existing files should be present", manifest.hasExistingFiles()); - Assert.assertEquals( - "Existing files count should match", 3, (int) manifest.existingFilesCount()); - Assert.assertEquals( - "Existing rows count should match", 26L, (long) manifest.existingRowsCount()); - - Assert.assertTrue("Deleted files should be present", manifest.hasDeletedFiles()); - Assert.assertEquals("Deleted files count should match", 2, (int) manifest.deletedFilesCount()); - Assert.assertEquals("Deleted rows count should match", 7L, (long) manifest.deletedRowsCount()); + testManifestWritesWithGivenCodec("GZIP"); } @Test @@ -359,6 +338,51 @@ public void testRollingDeleteManifestWriterSplitFiles() throws IOException { deletedRowCounts); } + @Test + public void testManifestWritesCompressedWithUncompressed() throws IOException { + testManifestWritesWithGivenCodec("UNCOMPRESSED"); + } + + @Test + public void testManifestWritesCompressedWithSnappy() throws IOException { + testManifestWritesWithGivenCodec("SNAPPY"); + } + + @Test + public void testManifestWritesCompressedWithZSTD() throws IOException { + testManifestWritesWithGivenCodec("ZSTD"); + } + + public void testManifestWritesWithGivenCodec(String codec) throws IOException { + Map config = ImmutableMap.of(AVRO_COMPRESSION, codec); + ManifestFile manifest = + writeManifest( + null, + "manifest.avro", + config, + manifestEntry(Status.ADDED, null, newFile(10)), + manifestEntry(Status.ADDED, null, newFile(20)), + manifestEntry(Status.ADDED, null, newFile(5)), + manifestEntry(Status.ADDED, null, newFile(5)), + manifestEntry(Status.EXISTING, null, newFile(15)), + manifestEntry(Status.EXISTING, null, newFile(10)), + manifestEntry(Status.EXISTING, null, newFile(1)), + manifestEntry(Status.DELETED, null, newFile(5)), + manifestEntry(Status.DELETED, null, newFile(2))); + + Assertions.assertThat(manifest.hasAddedFiles()).isTrue(); + Assertions.assertThat((int) manifest.addedFilesCount()).isEqualTo(4); + Assertions.assertThat((long) manifest.addedRowsCount()).isEqualTo(40L); + + Assertions.assertThat(manifest.hasExistingFiles()).isTrue(); + Assertions.assertThat((int) manifest.existingFilesCount()).isEqualTo(3); + Assertions.assertThat((long) manifest.existingRowsCount()).isEqualTo(26L); + + Assertions.assertThat(manifest.hasDeletedFiles()).isTrue(); + Assertions.assertThat((int) manifest.deletedFilesCount()).isEqualTo(2); + Assertions.assertThat((long) manifest.deletedRowsCount()).isEqualTo(7L); + } + private void checkManifests( List manifests, int[] addedFileCounts, diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d20ea9e50db9..cbe2efef3f8e 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -66,6 +66,8 @@ sqlite-jdbc = "3.41.0.0" testcontainers = "1.17.6" tez010 = "0.10.2" tez08 = { strictly = "[0.8, 0.9[", prefer = "0.8.4"} # see rich version usage explanation above +snappy = "1.1.10.3" +zstd = "1.5.5-5" [libraries] activation = { module = "javax.activation:activation", version.ref = "activation" } @@ -191,4 +193,6 @@ testcontainers = { module = "org.testcontainers:testcontainers", version.ref = " tez010-dag = { module = "org.apache.tez:tez-dag", version.ref = "tez010" } tez010-mapreduce = { module = "org.apache.tez:tez-mapreduce", version.ref = "tez010" } tez08-dag = { module = "org.apache.tez:tez-dag", version.ref = "tez08" } -tez08-mapreduce = { module = "org.apache.tez:tez-mapreduce", version.ref = "tez08" } \ No newline at end of file +tez08-mapreduce = { module = "org.apache.tez:tez-mapreduce", version.ref = "tez08" } +snappy = { module = "org.xerial.snappy:snappy-java", version.ref = "snappy" } +zstd = { module = "com.github.luben:zstd-jni", version.ref = "zstd" } \ No newline at end of file