Merge 4be7aab into e10e815

HDFGroup · Sep 30, 2021 · 850c720 · 850c720
2 parents e10e815 + 4be7aab
commit 850c720
Show file tree

Hide file tree

Showing 21 changed files with 708 additions and 59 deletions.
diff --git a/adapter/src/hermes/adapter/stdio/common/constants.h b/adapter/src/hermes/adapter/stdio/common/constants.h
@@ -16,10 +16,12 @@
 /**
  * Standard header
  */
+#include <cstdlib>
 
 /**
  * Dependent library header
  */
+#include "glog/logging.h"
 
 /**
  * Internal header
@@ -35,10 +37,29 @@ using hermes::adapter::stdio::MapperType;
  * Which mapper to be used by STDIO adapter.
  */
 const MapperType kMapperType = MapperType::BALANCED;
+
 /**
  * Define kPageSize for balanced mapping.
  */
-const size_t kPageSize = 1024 * 1024;
+const size_t kPageSize = []() {
+  const char *kPageSizeVar = "HERMES_PAGE_SIZE";
+  const size_t kDefaultPageSize = 1 * 1024 * 1024;
+
+  size_t result = kDefaultPageSize;
+  char *page_size = getenv(kPageSizeVar);
+
+  if (page_size) {
+    result = (size_t)std::strtoull(page_size, NULL, 0);
+    if (result == 0) {
+      LOG(FATAL) << "Invalid value of " << kPageSizeVar << ": " << page_size;
+    }
+  }
+
+  LOG(INFO) << "Stdio adapter page size: " << result << "\n";
+
+  return result;
+}();
+
 /**
  * String delimiter
  */

diff --git a/adapter/src/hermes/adapter/stdio/stdio.cc b/adapter/src/hermes/adapter/stdio/stdio.cc
@@ -248,7 +248,16 @@ void ReadGap(const std::string &filename, size_t seek_offset, u8 *read_ptr,
 void PutWithStdioFallback(AdapterStat &stat, const std::string &blob_name,
                           const std::string &filename, u8 *data, size_t size,
                           size_t offset) {
-  hapi::Status status = stat.st_bkid->Put(blob_name, data, size);
+  hapi::Context ctx;
+  const char *hermes_write_only = getenv("HERMES_WRITE_ONLY");
+
+  if (hermes_write_only && hermes_write_only[0] == '1') {
+    // Custom DPE for write-only apps like VPIC
+    ctx.rr_retry = true;
+    ctx.disable_swap = true;
+  }
+
+  hapi::Status status = stat.st_bkid->Put(blob_name, data, size, ctx);
   if (status.Failed()) {
     LOG(WARNING) << "Failed to Put Blob " << blob_name << " to Bucket "
                  << filename << ". Falling back to stdio." << std::endl;

diff --git a/benchmarks/CMakeLists.txt b/benchmarks/CMakeLists.txt
@@ -3,7 +3,7 @@ include_directories(
   ${PROJECT_SOURCE_DIR}/test
 )
 
-set(BENCHMARKS mdm_bench dpe_bench)
+set(BENCHMARKS mdm_bench dpe_bench vpic_bench)
 
 foreach(benchmark ${BENCHMARKS})
   add_executable(${benchmark} ${benchmark}.cc)

diff --git a/benchmarks/dpe_bench.cc b/benchmarks/dpe_bench.cc
@@ -140,7 +140,7 @@ int main(int argc, char **argv) {
     case api::PlacementPolicy::kRoundRobin: {
       time_point start_tm = now();
       result = RoundRobinPlacement(blob_sizes, tgt_state.bytes_available,
-                                   output_tmp, targets);
+                                   output_tmp, targets, false);
       std::cout << "DPE benchmark uses RoundRobin placement.\n\n";
       time_point end_tm = now();
       dpe_seconds = std::chrono::duration<double>(end_tm - start_tm).count();