issue #2121 - skip file creation when the buffer is empty

Also, set more status fields in transient user data (chunkData) to facilitate logging / debug. Signed-off-by: Lee Surprenant <lmsurpre@us.ibm.com>
LinuxForHealth · Mar 22, 2021 · 015a36f · 015a36f
1 parent 1cbcff8
commit 015a36f
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 9 deletions.
diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/audit/BulkAuditLogger.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/audit/BulkAuditLogger.java
@@ -54,8 +54,8 @@ public BulkAuditLogger() {
 
     public boolean shouldLog() {
         // Wraps common code for logging
-        if (log.isLoggable(Level.FINE)) {
-            log.fine("Bulk Data Audit Log is '" + svc.isEnabled() + "'");
+        if (log.isLoggable(Level.FINER)) {
+            log.finer("Bulk Data Audit Log is '" + svc.isEnabled() + "'");
         }
         return svc.isEnabled();
     }

diff --git a/...c/main/java/com/ibm/fhir/bulkdata/jbatch/export/checkpoint/ExportCheckpointAlgorithm.java b/...c/main/java/com/ibm/fhir/bulkdata/jbatch/export/checkpoint/ExportCheckpointAlgorithm.java
@@ -66,6 +66,7 @@ public void beginCheckpoint() {
             if (chunkData != null) {
                 logger.fine("begin checkpoint [" +
                         "page " + chunkData.getPageNum() + " of " + chunkData.getLastPageNum() + ", " +
+                        "uploadCount=" + chunkData.getUploadCount() + ", " +
                         "bufferSize=" + chunkData.getBufferStream().size() + ", " +
                         "uploadPart=" + chunkData.getPartNum() + ", " +
                         "currentUploadSize=" + chunkData.getCurrentUploadSize() + ", " +
@@ -84,6 +85,7 @@ public void endCheckpoint() {
             } else {
                 logger.fine("end checkpoint [" +
                         "page " + chunkData.getPageNum() + " of " + chunkData.getLastPageNum() + ", " +
+                        "uploadCount=" + chunkData.getUploadCount() + ", " +
                         "bufferSize=" + chunkData.getBufferStream().size() + ", " +
                         "uploadPart=" + chunkData.getPartNum() + ", " +
                         "currentUploadSize=" + chunkData.getCurrentUploadSize() + ", " +
@@ -111,10 +113,21 @@ public boolean isReadyToCheckpoint() {
         boolean readyToWrite = chunkData.getBufferStream().size() >= writeTrigger;
 
         // Check if we should finish writing the current object/file
-        boolean overFileSizeThreshold = sizeThreshold != 0 && chunkData.getBufferStream().size() >= sizeThreshold;
-        boolean overMaxResourceCountThreshold = resourceCountThreshold != 0 && chunkData.getCurrentUploadResourceNum() >= resourceCountThreshold;
+        boolean overSizeThreshold = sizeThreshold != 0 && chunkData.getCurrentUploadSize() >= sizeThreshold;
+        boolean overResourceCountThreshold = resourceCountThreshold != 0 && chunkData.getCurrentUploadResourceNum() >= resourceCountThreshold;
         boolean end = chunkData.getPageNum() >= chunkData.getLastPageNum();
-        chunkData.setFinishCurrentUpload(overFileSizeThreshold || overMaxResourceCountThreshold || end);
+        chunkData.setFinishCurrentUpload(overSizeThreshold || overResourceCountThreshold || end);
+
+        if (logger.isLoggable(Level.FINE)) {
+            logger.fine("isReadyToCheckpoint [" +
+                    "readyToWrite=" + readyToWrite + " (" + chunkData.getBufferStream().size() + " >= " + writeTrigger + "), " +
+                    "overSizeThreshold=" + overSizeThreshold +
+                            " (" + chunkData.getCurrentUploadSize() + " >= " + sizeThreshold + "), " +
+                    "overResourceCountThreshold=" + overResourceCountThreshold +
+                            " (" + chunkData.getCurrentUploadResourceNum() + " >= " + resourceCountThreshold + "), " +
+                    "end=" + end + " (" + chunkData.getPageNum() + " >= " + chunkData.getLastPageNum() + ") " +
+                    "]");
+        }
 
         // There are two conditions that trigger a checkpoint:
         // 1 - We have enough bytes to start writing a part; or

diff --git a/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/provider/impl/FileProvider.java b/fhir-bulkdata-webapp/src/main/java/com/ibm/fhir/bulkdata/provider/impl/FileProvider.java
@@ -159,6 +159,12 @@ public void writeResources(String mediaType, List<ReadResultDTO> dtos) throws Ex
         if (!FHIRMediaType.APPLICATION_NDJSON.equals(mediaType)) {
             throw new UnsupportedOperationException("FileProvider does not support writing files of type " + mediaType);
         }
+        if (chunkData.getBufferStream().size() == 0) {
+            // Early exit condition:  nothing to write so just set the latWrittenPageNum and return
+            chunkData.setLastWrittenPageNum(chunkData.getPageNum());
+            return;
+        }
+
         if (out == null) {
             this.fileName = exportPathPrefix + File.separator + fhirResourceType + "_" + chunkData.getUploadCount() + ".ndjson";
             String base = configuration.getBaseFileLocation(source);
@@ -197,8 +203,15 @@ public void writeResources(String mediaType, List<ReadResultDTO> dtos) throws Ex
         if (chunkData.isFinishCurrentUpload()) {
             out.close();
             out = null;
+            chunkData.setPartNum(1);
+            chunkData.setCurrentUploadResourceNum(0);
+            chunkData.setCurrentUploadSize(0);
+            chunkData.setFinishCurrentUpload(false);
             chunkData.setUploadCount(chunkData.getUploadCount() + 1);
-            chunkData.getBufferStream().reset();
+        } else {
+            chunkData.setPartNum(chunkData.getPartNum() + 1);
         }
+
+        chunkData.setLastWrittenPageNum(chunkData.getPageNum());
     }
 }
diff --git a/...ain/java/com/ibm/fhir/operation/bulkdata/config/impl/AbstractSystemConfigurationImpl.java b/...ain/java/com/ibm/fhir/operation/bulkdata/config/impl/AbstractSystemConfigurationImpl.java
@@ -173,7 +173,7 @@ public int getCoreFileWriteTriggerSize() {
     }
 
     private static final int defaultCoreFileWriteTriggerSize() {
-        return 1024 * 1024 * FHIRConfigHelper.getIntProperty("fhirServer/bulkdata/core/cos/partUploadTriggerSizeMB", DEFAULT_FILE_WRITE_TRIGGER_SIZE_MB);
+        return 1024 * 1024 * FHIRConfigHelper.getIntProperty("fhirServer/bulkdata/core/file/writeTriggerSizeMB", DEFAULT_FILE_WRITE_TRIGGER_SIZE_MB);
     }
 
     @Override
@@ -182,7 +182,7 @@ public long getCoreFileSizeThreshold() {
     }
 
     private static final long defaultCoreFileSizeThreshold() {
-        final String PATH = "fhirServer/bulkdata/core/cos/objectSizeThresholdMB";
+        final String PATH = "fhirServer/bulkdata/core/file/sizeThresholdMB";
         return 1024l * 1024l * FHIRConfigHelper.getIntProperty(PATH, DEFAULT_FILE_MAX_SIZE_MB);
     }
 
@@ -192,7 +192,7 @@ public int getCoreFileResourceCountThreshold() {
     }
 
     private static final int defaultCoreFileResourceCountThreshold() {
-        final String PATH = "fhirServer/bulkdata/core/cos/objectResourceCountThreshold";
+        final String PATH = "fhirServer/bulkdata/core/file/resourceCountThreshold";
         return FHIRConfigHelper.getIntProperty(PATH, DEFAULT_FILE_MAX_RESOURCE_COUNT);
     }