From b1a3c6eb8bbabf29e95a20d866b9a3c83681ad48 Mon Sep 17 00:00:00 2001 From: sivabalan Date: Fri, 29 Jul 2022 11:11:20 +0530 Subject: [PATCH 1/3] throwing exception when restore is attempted with hoodie.arhive.beyond.savepoint is enabled --- .../hudi/client/BaseHoodieWriteClient.java | 3 + .../apache/hudi/config/HoodieWriteConfig.java | 5 ++ .../TestHoodieClientOnCopyOnWriteStorage.java | 56 +++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index ab14f2216a5a..362ded0b4865 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -61,6 +61,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.ValidationUtils; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieArchivalConfig; import org.apache.hudi.config.HoodieClusteringConfig; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieWriteConfig; @@ -721,6 +722,8 @@ public void restoreToSavepoint(String savepointTime) { HoodieTable table = initTable(WriteOperationType.UNKNOWN, Option.empty(), initialMetadataTableIfNecessary); SavepointHelpers.validateSavepointPresence(table, savepointTime); + ValidationUtils.checkArgument(config.shouldArchiveBeyondSavepoint(), "Restore is not supported when " + HoodieArchivalConfig.ARCHIVE_BEYOND_SAVEPOINT.key() + + " is enabled"); restoreToInstant(savepointTime, initialMetadataTableIfNecessary); SavepointHelpers.validateSavepointRestore(table, savepointTime); } diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 2cb9e8ce8271..93d40b495791 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -2447,6 +2447,11 @@ public Builder withAllowMultiWriteOnSameInstant(boolean allow) { return this; } + public Builder withArchiveBeyondSavepoint(boolean allow) { + writeConfig.setValue(HoodieArchivalConfig.ARCHIVE_BEYOND_SAVEPOINT, String.valueOf(allow)); + return this; + } + public Builder withExternalSchemaTrasformation(boolean enabled) { writeConfig.setValue(AVRO_EXTERNAL_SCHEMA_TRANSFORMATION_ENABLE, String.valueOf(enabled)); return this; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java index 8ba459b77226..028e572a9337 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java @@ -71,6 +71,7 @@ import org.apache.hudi.common.util.Option; import org.apache.hudi.common.util.StringUtils; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.config.HoodieArchivalConfig; import org.apache.hudi.config.HoodieCompactionConfig; import org.apache.hudi.config.HoodieIndexConfig; import org.apache.hudi.config.HoodieClusteringConfig; @@ -675,6 +676,61 @@ private void testUpsertsInternal(HoodieWriteConfig config, }).collect(); } + @Test + public void testRestoreWithSavepointBeyondArchival() throws Exception { + HoodieWriteConfig config = getConfigBuilder().withRollbackUsingMarkers(true).build(); + HoodieWriteConfig hoodieWriteConfig = getConfigBuilder(EAGER) + .withRollbackUsingMarkers(true) + .withArchiveBeyondSavepoint(true) + .withProps(config.getProps()).withTimelineLayoutVersion( + VERSION_0).build(); + + HoodieTableMetaClient.withPropertyBuilder() + .fromMetaClient(metaClient) + .setTimelineLayoutVersion(VERSION_0) + .setPopulateMetaFields(config.populateMetaFields()) + .initTable(metaClient.getHadoopConf(), metaClient.getBasePath()); + + SparkRDDWriteClient client = getHoodieWriteClient(hoodieWriteConfig); + + // Write 1 (only inserts) + String newCommitTime = "001"; + String initCommitTime = "000"; + int numRecords = 200; + insertFirstBatch(hoodieWriteConfig, client, newCommitTime, initCommitTime, numRecords, SparkRDDWriteClient::insert, + false, true, numRecords, config.populateMetaFields()); + + // Write 2 (updates) + String prevCommitTime = newCommitTime; + newCommitTime = "004"; + numRecords = 100; + String commitTimeBetweenPrevAndNew = "002"; + updateBatch(hoodieWriteConfig, client, newCommitTime, prevCommitTime, + Option.of(Arrays.asList(commitTimeBetweenPrevAndNew)), initCommitTime, numRecords, SparkRDDWriteClient::upsert, false, true, + numRecords, 200, 2, config.populateMetaFields()); + + // Delete 1 + prevCommitTime = newCommitTime; + newCommitTime = "005"; + numRecords = 50; + + deleteBatch(hoodieWriteConfig, client, newCommitTime, prevCommitTime, + initCommitTime, numRecords, SparkRDDWriteClient::delete, false, true, + 0, 150, config.populateMetaFields()); + + HoodieWriteConfig newConfig = getConfigBuilder().withProps(config.getProps()).withTimelineLayoutVersion( + TimelineLayoutVersion.CURR_VERSION).build(); + client = getHoodieWriteClient(newConfig); + + client.savepoint("004", "user1", "comment1"); + + // verify that restore fails when "hoodie.archive.beyond.savepoint" is enabled. + SparkRDDWriteClient finalClient = client; + assertThrows(IllegalArgumentException.class, () -> { + finalClient.restoreToSavepoint("004"); + }, "Restore should not be supported when " + HoodieArchivalConfig.ARCHIVE_BEYOND_SAVEPOINT.key() + " is enabled"); + } + /** * Test Insert API for HoodieConcatHandle. */ From bff48a42c6a9c10715dc0b2b4958cd2b904ec82d Mon Sep 17 00:00:00 2001 From: sivabalan Date: Mon, 1 Aug 2022 10:49:30 -0700 Subject: [PATCH 2/3] addressing comments --- .../main/java/org/apache/hudi/config/HoodieWriteConfig.java | 5 ----- .../functional/TestHoodieClientOnCopyOnWriteStorage.java | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java index 93d40b495791..2cb9e8ce8271 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java @@ -2447,11 +2447,6 @@ public Builder withAllowMultiWriteOnSameInstant(boolean allow) { return this; } - public Builder withArchiveBeyondSavepoint(boolean allow) { - writeConfig.setValue(HoodieArchivalConfig.ARCHIVE_BEYOND_SAVEPOINT, String.valueOf(allow)); - return this; - } - public Builder withExternalSchemaTrasformation(boolean enabled) { writeConfig.setValue(AVRO_EXTERNAL_SCHEMA_TRANSFORMATION_ENABLE, String.valueOf(enabled)); return this; diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java index 028e572a9337..faa8b63114ce 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java @@ -681,7 +681,7 @@ public void testRestoreWithSavepointBeyondArchival() throws Exception { HoodieWriteConfig config = getConfigBuilder().withRollbackUsingMarkers(true).build(); HoodieWriteConfig hoodieWriteConfig = getConfigBuilder(EAGER) .withRollbackUsingMarkers(true) - .withArchiveBeyondSavepoint(true) + .withArchivalConfig(HoodieArchivalConfig.newBuilder().withArchiveBeyondSavepoint(true).build()) .withProps(config.getProps()).withTimelineLayoutVersion( VERSION_0).build(); From 7df35ac1c98d2aa8c95e42ab00c1fe514997df1a Mon Sep 17 00:00:00 2001 From: sivabalan Date: Tue, 2 Aug 2022 07:46:31 -0700 Subject: [PATCH 3/3] fixing tests --- .../java/org/apache/hudi/client/BaseHoodieWriteClient.java | 2 +- .../functional/TestHoodieClientOnCopyOnWriteStorage.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java index 362ded0b4865..4cf58f1c3009 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/client/BaseHoodieWriteClient.java @@ -722,7 +722,7 @@ public void restoreToSavepoint(String savepointTime) { HoodieTable table = initTable(WriteOperationType.UNKNOWN, Option.empty(), initialMetadataTableIfNecessary); SavepointHelpers.validateSavepointPresence(table, savepointTime); - ValidationUtils.checkArgument(config.shouldArchiveBeyondSavepoint(), "Restore is not supported when " + HoodieArchivalConfig.ARCHIVE_BEYOND_SAVEPOINT.key() + ValidationUtils.checkArgument(!config.shouldArchiveBeyondSavepoint(), "Restore is not supported when " + HoodieArchivalConfig.ARCHIVE_BEYOND_SAVEPOINT.key() + " is enabled"); restoreToInstant(savepointTime, initialMetadataTableIfNecessary); SavepointHelpers.validateSavepointRestore(table, savepointTime); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java index faa8b63114ce..28108b793aca 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/functional/TestHoodieClientOnCopyOnWriteStorage.java @@ -719,7 +719,8 @@ public void testRestoreWithSavepointBeyondArchival() throws Exception { 0, 150, config.populateMetaFields()); HoodieWriteConfig newConfig = getConfigBuilder().withProps(config.getProps()).withTimelineLayoutVersion( - TimelineLayoutVersion.CURR_VERSION).build(); + TimelineLayoutVersion.CURR_VERSION) + .withArchivalConfig(HoodieArchivalConfig.newBuilder().withArchiveBeyondSavepoint(true).build()).build(); client = getHoodieWriteClient(newConfig); client.savepoint("004", "user1", "comment1");