Checking storage level before persisting preppedRecords

apache · Mar 22, 2018 · 855c34b · 855c34b
1 parent 123da02
commit 855c34b
Showing 1 changed file with 6 additions and 1 deletion.
diff --git a/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java b/hoodie-client/src/main/java/com/uber/hoodie/HoodieWriteClient.java
@@ -414,7 +414,12 @@ private JavaRDD<WriteStatus> upsertRecordsInternal(JavaRDD<HoodieRecord<T>> prep
       final boolean isUpsert) {
 
     // Cache the tagged records, so we don't end up computing both
-    preppedRecords.persist(StorageLevel.MEMORY_AND_DISK_SER());
+    // TODO: Consistent contract in HoodieWriteClient regarding preppedRecord storage level handling
+    if (preppedRecords.getStorageLevel() == StorageLevel.NONE()) {
+      preppedRecords.persist(StorageLevel.MEMORY_AND_DISK_SER());
+    } else {
+      logger.info("RDD PreppedRecords was persisted at: " + preppedRecords.getStorageLevel());
+    }
 
     WorkloadProfile profile = null;
     if (hoodieTable.isWorkloadProfileNeeded()) {