Merge pull request #68 from hariharan-devarajan/hariharan/posix_adapter

Hariharan/posix adapter

.Dockerfile

@@ -0,0 +1,64 @@
+FROM gitpod/workspace-full
+
+USER gitpod
+RUN sudo apt-get update -q && \
+    sudo apt-get install -yq gcc g++
+
+RUN sudo apt-get install -y --no-install-recommends \
+    autoconf \
+    ca-certificates \
+    curl \
+    environment-modules \
+    git \
+    build-essential \
+    python \
+    nano \
+    sudo \
+    unzip \
+    cmake \
+    mpich
+
+ENV LOCAL=$HOME
+ENV PROJECT=$LOCAL/hermes
+ENV INSTALL_DIR=$LOCAL/install
+ENV SPACK_DIR=$LOCAL/spack
+ENV SDS_DIR=$LOCAL/sds
+
+RUN mkdir -p $INSTALL_DIR
+
+RUN git clone https://github.com/spack/spack ${SPACK_DIR}
+RUN git clone https://xgitlab.cels.anl.gov/sds/sds-repo.git ${SDS_DIR}
+RUN git clone https://github.com/HDFGroup/hermes ${PROJECT}
+
+ENV spack=${SPACK_DIR}/bin/spack
+
+RUN . ${SPACK_DIR}/share/spack/setup-env.sh
+
+RUN echo $GITPOD_REPO_ROOT
+
+RUN $spack repo add ${SDS_DIR}
+RUN $spack repo add $PROJECT/ci/hermes
+
+RUN $spack compiler find
+
+RUN $spack compiler list
+
+ENV THALLIUM_VERSION=0.8.3 \
+    GOTCHA_VERSION=develop \
+    CATCH2_VERSION=2.13.3
+
+ENV GOTCHA_SPEC=gotcha@${GOTCHA_VERSION}
+RUN $spack install ${GOTCHA_SPEC}
+ENV THALLIUM_SPEC="mochi-thallium~cereal@${THALLIUM_VERSION} ^mercury~boostsys"
+RUN $spack install ${THALLIUM_SPEC}
+ENV CATCH2_SPEC="catch2@${CATCH2_VERSION}"
+RUN $spack install ${CATCH2_SPEC}
+
+RUN $spack view --verbose symlink ${INSTALL_DIR} ${THALLIUM_SPEC} ${GOTCHA_SPEC} ${CATCH2_SPEC}
+
+ENV ORTOOLS_VERSION=v7.7
+ENV ORTOOLS_MINOR_VERSION=7810
+ENV ORTOOLS_BASE_URL=https://github.com/google/or-tools/releases/download
+ENV ORTOOLS_TARBALL_NAME=or-tools_ubuntu-18.04_${ORTOOLS_VERSION}.${ORTOOLS_MINOR_VERSION}.tar.gz
+RUN wget ${ORTOOLS_BASE_URL}/${ORTOOLS_VERSION}/${ORTOOLS_TARBALL_NAME}
+RUN tar -xvf ${ORTOOLS_TARBALL_NAME} -C ${INSTALL_DIR} --strip-components=1

.clang-format

@@ -0,0 +1 @@
+BasedOnStyle: Google

.gitignore

@@ -9,4 +9,7 @@ GTAGS
 /cmake-build-debug/
 /.idea/
 /.clang-format
-__pycache__/
+__pycache__/
+/cmake-build-debug-system/
+/src/adapter/posix/cmake-build-debug-system/CMakeFiles/clion-log.txt
+/adapter/test/posix/Testing/Temporary/LastTest.log

CMakeLists.txt

@@ -124,8 +124,7 @@ if(NOT "${CMAKE_CXX_STANDARD}")
 endif()
 
 if(NOT CMAKE_CXX_FLAGS)
-#TODO: add -Werror
- set (CMAKE_CXX_FLAGS "-Wall -Wextra ${CMAKE_CXX_FLAGS} -DCATCH_CONFIG_FAST_COMPILE")
+ set (CMAKE_CXX_FLAGS "-Wall -Wextra -Werror ${CMAKE_CXX_FLAGS}")
  message (STATUS "Warnings Configuration: default: ${CMAKE_CXX_FLAGS}")
 endif()
 
@@ -153,6 +152,7 @@ option(HERMES_DEBUG_HEAP "Store Heap debug information for visualization." OFF)
 option(HERMES_BUILD_BENCHMARKS "Build the Hermes benchmark suite." OFF)
 option(HERMES_ENABLE_TIMING "Turn on timing of selected functions." OFF)
 option(HERMES_ENABLE_COVERAGE "Enable code coverage." OFF)
+option(HERMES_ENABLE_ADAPTERS "Enable hermes adapters." ON)
 # Calculate code coverage with debug mode
 if(HERMES_ENABLE_COVERAGE)
   if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
@@ -330,7 +330,9 @@ if(HERMES_HAVE_GOTCHA)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/gotcha_intercept)
 endif()
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src)
-#add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/posix_adapter)
+if(HERMES_ENABLE_ADAPTERS)
+  add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/adapter)
+endif()
 #add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/mpi_io_adapter)
 #add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/hermes_vol)
 
@@ -349,6 +351,9 @@ endif()
 if(CMAKE_PROJECT_NAME STREQUAL HERMES AND BUILD_TESTING)
   find_package(Catch2 REQUIRED)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/test)
+  if(HERMES_ENABLE_ADAPTERS)
+    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/adapter/test)
+  endif()
 endif()
 
 #-----------------------------------------------------------------------------

adapter/CMakeLists.txt

@@ -0,0 +1,12 @@
+# Set hermes to preload if adapter is linked.
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHERMES_PRELOAD -DHERMES_RPC_THALLIUM")
+
+# include directory for absolute references to files and headers.
+include_directories(${CMAKE_SOURCE_DIR}/adapter/include
+        ${CMAKE_SOURCE_DIR}/adapter/src)
+
+# add stdio adapter
+add_subdirectory(src/hermes/adapter/stdio)
+
+# add posix adapter
+add_subdirectory(src/hermes/adapter/posix)

adapter/include/hermes/adapter/posix.h

@@ -0,0 +1,17 @@
+//
+// Created by hdevarajan on 12/7/20.
+//
+
+#ifndef HERMES_ADAPTER_POSIX_H
+#define HERMES_ADAPTER_POSIX_H
+
+namespace hermes::adapter {
+class Posix {
+  /**
+   *
+   */
+  Posix();
+};
+}  // namespace hermes::adapter
+
+#endif  // HERMES_ADAPTER_POSIX_H

adapter/include/hermes/adapter/stdio.h

@@ -0,0 +1,72 @@
+#ifndef HERMES_ADAPTER_STDIO_H
+#define HERMES_ADAPTER_STDIO_H
+
+/**
+ * Standard header
+ */
+#include <fcntl.h>
+#include <stdarg.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#include <experimental/filesystem>
+
+/**
+ * Dependent library headers
+ */
+#include "glog/logging.h"
+
+/**
+ * Internal headers
+ */
+#include <bucket.h>
+#include <hermes.h>
+#include <hermes/adapter/interceptor.h>
+#include <hermes/adapter/singleton.h>
+#include <hermes/adapter/stdio/common/constants.h>
+#include <hermes/adapter/stdio/common/datastructures.h>
+#include <hermes/adapter/stdio/mapper/mapper_factory.h>
+#include <hermes/adapter/stdio/metadata_manager.h>
+#include <vbucket.h>
+
+/**
+ * Function declarations
+ */
+HERMES_FORWARD_DECL(ftell, long int, (FILE * fp));
+HERMES_FORWARD_DECL(fopen, FILE *, (const char *path, const char *mode));
+HERMES_FORWARD_DECL(fopen64, FILE *, (const char *path, const char *mode));
+HERMES_FORWARD_DECL(fdopen, FILE *, (int fd, const char *mode));
+HERMES_FORWARD_DECL(freopen, FILE *,
+                    (const char *path, const char *mode, FILE *stream));
+HERMES_FORWARD_DECL(freopen64, FILE *,
+                    (const char *path, const char *mode, FILE *stream));
+HERMES_FORWARD_DECL(fclose, int, (FILE * fp));
+HERMES_FORWARD_DECL(fflush, int, (FILE * fp));
+HERMES_FORWARD_DECL(fwrite, size_t,
+                    (const void *ptr, size_t size, size_t nmemb, FILE *stream));
+HERMES_FORWARD_DECL(fputc, int, (int c, FILE *stream));
+HERMES_FORWARD_DECL(putc, int, (int c, FILE *stream));
+HERMES_FORWARD_DECL(fgetpos, int, (FILE * stream, fpos_t *pos));
+HERMES_FORWARD_DECL(putw, int, (int w, FILE *stream));
+HERMES_FORWARD_DECL(fputs, int, (const char *s, FILE *stream));
+HERMES_FORWARD_DECL(fprintf, int, (FILE * stream, const char *format, ...));
+HERMES_FORWARD_DECL(printf, int, (const char *format, ...));
+HERMES_FORWARD_DECL(vfprintf, int,
+                    (FILE * stream, const char *format, va_list));
+HERMES_FORWARD_DECL(vprintf, int, (const char *format, va_list));
+HERMES_FORWARD_DECL(fread, size_t,
+                    (void *ptr, size_t size, size_t nmemb, FILE *stream));
+HERMES_FORWARD_DECL(fgetc, int, (FILE * stream));
+HERMES_FORWARD_DECL(getc, int, (FILE * stream));
+HERMES_FORWARD_DECL(getw, int, (FILE * stream));
+HERMES_FORWARD_DECL(_IO_getc, int, (FILE * stream));
+HERMES_FORWARD_DECL(_IO_putc, int, (int, FILE *stream));
+HERMES_FORWARD_DECL(fgets, char *, (char *s, int size, FILE *stream));
+HERMES_FORWARD_DECL(fseek, int, (FILE * stream, long offset, int whence));
+HERMES_FORWARD_DECL(fseeko, int, (FILE * stream, off_t offset, int whence));
+HERMES_FORWARD_DECL(fseeko64, int, (FILE * stream, off64_t offset, int whence));
+HERMES_FORWARD_DECL(fsetpos, int, (FILE * stream, const fpos_t *pos));
+HERMES_FORWARD_DECL(fsetpos64, int, (FILE * stream, const fpos64_t *pos));
+HERMES_FORWARD_DECL(rewind, void, (FILE * stream));
+
+#endif  // HERMES_ADAPTER_STDIO_H

adapter/src/hermes/adapter/constants.h

@@ -0,0 +1,20 @@
+#ifndef HERMES_ADAPTER_CONSTANTS_H
+#define HERMES_ADAPTER_CONSTANTS_H
+
+/**
+ * Constants file for all adapter.
+ * This file contains common constants across different adapters.
+ */
+
+/**
+ * kHermesConf env variable is used to define path to kHermesConf in adapters.
+ * This is used for initialization of Hermes.
+ */
+const char* kHermesConf = "HERMES_CONF";
+/**
+ * kHermesExtension is the extension of hermes files. These are buffered files
+ * from I/O. This is needed as we should not intercept these files.
+ */
+const char* kHermesExtension = ".hermes";
+
+#endif  // HERMES_ADAPTER_CONSTANTS_H

adapter/src/hermes/adapter/interceptor.cc

@@ -0,0 +1,90 @@
+/**
+ * Internal headers
+ */
+#include "interceptor.h"
+
+namespace hermes::adapter {
+/**
+ * is exit by program called.
+ */
+bool exit = false;
+
+void PopulateBufferingPath() {
+  char* hermes_config = getenv(kHermesConf);
+  hermes::Config config = {};
+  const size_t kConfigMemorySize = KILOBYTES(16);
+  hermes::u8 config_memory[kConfigMemorySize];
+  if (hermes_config && strlen(hermes_config) > 0) {
+    INTERCEPTOR_LIST->hermes_paths_exclusion.push_back(hermes_config);
+    hermes::Arena config_arena = {};
+    hermes::InitArena(&config_arena, kConfigMemorySize, config_memory);
+    hermes::ParseConfig(&config_arena, hermes_config, &config);
+  } else {
+    InitDefaultConfig(&config);
+  }
+
+  for (const auto& item : config.mount_points) {
+    if (!item.empty()) {
+      INTERCEPTOR_LIST->hermes_paths_exclusion.push_back(item);
+    }
+  }
+  INTERCEPTOR_LIST->hermes_paths_exclusion.push_back(
+      config.buffer_pool_shmem_name);
+  INTERCEPTOR_LIST->hermes_paths_exclusion.push_back(kHermesExtension);
+}
+bool IsTracked(const std::string& path) {
+  if (hermes::adapter::exit) return false;
+  atexit(OnExit);
+  if (INTERCEPTOR_LIST->hermes_paths_exclusion.empty()) {
+    PopulateBufferingPath();
+  }
+  for (const auto& pth : INTERCEPTOR_LIST->hermes_flush_exclusion) {
+    if (path.find(pth) != std::string::npos) {
+      return false;
+    }
+  }
+  for (const auto& pth : INTERCEPTOR_LIST->hermes_paths_exclusion) {
+    if (path.find(pth) != std::string::npos ||
+        pth.find(path) != std::string::npos) {
+      return false;
+    }
+  }
+  if (INTERCEPTOR_LIST->user_path_exclusions.empty()) {
+    for (const auto& pth : kPathInclusions) {
+      if (path.find(pth) == 0) {
+        return true;
+      }
+    }
+    for (const auto& pth : kPathExclusions) {
+      if (path.find(pth) == 0) {
+        return false;
+      }
+    }
+  } else {
+    for (const auto& pth : INTERCEPTOR_LIST->user_path_exclusions) {
+      if (path.find(pth) == 0) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+bool IsTracked(FILE* fh) {
+  if (hermes::adapter::exit) return false;
+  atexit(OnExit);
+  const int kMaxSize = 0xFFF;
+  char proclnk[kMaxSize];
+  char filename[kMaxSize];
+  int fno = fileno(fh);
+  snprintf(proclnk, kMaxSize, "/proc/self/fd/%d", fno);
+  size_t r = readlink(proclnk, filename, kMaxSize);
+  if (r > 0) {
+    std::string file_str(filename);
+    return IsTracked(file_str);
+  }
+  return false;
+}
+
+void OnExit(void) { hermes::adapter::exit = true; }
+
+}  // namespace hermes::adapter