Merge pull request #453 from lukemartinlogan/master

Stag-in/stage-out + major adaptor refactor

CMake/Testing/Temporary/CTestCostData.txt

@@ -0,0 +1 @@
+---

CMake/Testing/Temporary/LastTest.log

@@ -0,0 +1,3 @@
+Start testing: Sep 28 12:37 CDT
+----------------------------------------------------------
+End testing: Sep 28 12:37 CDT

CMakeLists.txt

@@ -399,6 +399,8 @@ if(HERMES_BUILD_BUFFER_POOL_VISUALIZER AND SDL2_FOUND)
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/buffer_pool_visualizer)
 endif()
 
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/data_stager)
+
 #-----------------------------------------------------------------------------
 # Testing
 #-----------------------------------------------------------------------------

adapter/Testing/Temporary/LastTest.log

@@ -0,0 +1,3 @@
+Start testing: Sep 28 12:36 CDT
+----------------------------------------------------------
+End testing: Sep 28 12:36 CDT

adapter/adapter_generator/README.md

@@ -0,0 +1,14 @@
+# Adapter Generator
+
+This library generates an interceptor "API" class used to store the original implementations
+of a library for an LD_PRELOAD interceptor to use.
+
+## Usage
+
+To create the interceptor for POSIX:
+```bash
+cd /path/to/adapter_generator
+python3 posix.py
+```
+
+This will create the file adapter/posix/posix.h

adapter/adapter_generator/adapter_generator/create_interceptor.py

@@ -0,0 +1,146 @@
+import sys,os
+import re
+
+preamble = """/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Distributed under BSD 3-Clause license.                                   *
+ * Copyright by The HDF Group.                                               *
+ * Copyright by the Illinois Institute of Technology.                        *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of Hermes. The full Hermes copyright notice, including  *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the COPYING file, which can be found at the top directory. If you do not  *
+ * have access to the file, you may request a copy from help@hdfgroup.org.   *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */"""
+
+class Api:
+    def __init__(self, api_str):
+        self.decompose_prototype(api_str)
+
+    def _is_text(self, tok):
+        first_is_text = re.match("[_a-zA-Z]", tok[0]) is not None
+        if not first_is_text:
+            return False
+        return re.match("[_a-zA-Z0-9]+", tok) is not None
+
+    def _clean(self, toks):
+        return [tok for tok in toks if tok is not None and len(tok) > 0]
+
+    def get_arg_tuple(self, arg):
+        arg_toks = self._clean(re.split("[ ]|(\*+)", arg))
+        if len(arg_toks) == 1:
+            if arg_toks[0] == '...':
+                type = ""
+                name = "..."
+                return (type, name)
+        type = " ".join(arg_toks[:-1])
+        name = arg_toks[-1]
+        return (type, name)
+
+    def decompose_prototype(self, api_str):
+        toks = self._clean(re.split("[()]", api_str))
+        proto, args = toks[0], toks[1]
+
+        try:
+            proto = self._clean(re.split("[ ]|(\*+)", proto))
+            self.name = proto[-1]
+            self.ret = " ".join(proto[:-1])
+            self.real_name = self.name
+            self.type = f"{self.name}_t"
+        except:
+            print(f"Failed to decompose proto name: {proto}")
+            exit()
+
+        try:
+            self.var_defs = []
+            args = args.split(',')
+            for arg in args:
+                self.var_defs.append(self.get_arg_tuple(arg))
+        except:
+            print(f"Failed to decompose proto args: {args}")
+            exit(1)
+
+    def get_args(self):
+        if len(self.var_defs) == 0:
+            return ""
+        try:
+            args = [" ".join(arg_tuple) for arg_tuple in self.var_defs]
+        except:
+            print(f"Failed to get arg list: {self.var_defs}")
+            exit(1)
+        return ", ".join(args)
+
+    def pass_args(self):
+        if self.var_defs is None:
+            return ""
+        args = [arg[-1] for arg in self.var_defs]
+        return ", ".join(args)
+
+class ApiClass:
+    def __init__(self, namespace, apis, includes, path=None, do_save=True):
+        self.apis = apis
+        self.lines = []
+
+        self.lines.append(preamble)
+        self.lines.append("")
+        self.lines.append(f"#ifndef HERMES_ADAPTER_{namespace.upper()}_H")
+        self.lines.append(f"#define HERMES_ADAPTER_{namespace.upper()}_H")
+
+        self.lines.append("#include <string>")
+        self.lines.append("#include <dlfcn.h>")
+        self.lines.append("#include <iostream>")
+        self.lines.append("#include <glog/logging.h>")
+        self.lines.append("#include \"interceptor.h\"")
+        self.lines.append("#include \"filesystem/filesystem.h\"")
+        for include in includes:
+            self.lines.append(f"#include {include}")
+        self.lines.append("")
+
+        self.lines.append(f"namespace hermes::adapter::{namespace} {{")
+        self.lines.append(f"")
+        self.lines.append(f"class API {{")
+
+        self.lines.append(f" public:")
+        for api in self.apis:
+            self.add_intercept_api(api)
+
+        self.lines.append(f"  API() {{")
+        self.lines.append(f"    void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, \"{namespace}_intercepted\");")
+        for api in self.apis:
+            self.init_api(api)
+        self.lines.append(f"  }}")
+        self.lines.append(f"}};")
+        self.lines.append(f"}}  // namespace hermes::adapter::{namespace}")
+
+        self.lines.append("")
+        self.lines.append(f"#endif  // HERMES_ADAPTER_{namespace.upper()}_H")
+        self.lines.append("")
+        self.text = "\n".join(self.lines)
+
+        if do_save:
+            self.save(path, namespace)
+        else:
+            print(self.text)
+
+
+    def save(self, path, namespace):
+        if path is None:
+            ns_dir = os.path.dirname(os.getcwd())
+            path = os.path.join(ns_dir, namespace, f"real_api.h")
+        with open(path, "w") as fp:
+            fp.write(self.text)
+
+    def add_intercept_api(self, api):
+        self.lines.append(f"  typedef {api.ret} (*{api.type})({api.get_args()});")
+        self.lines.append(f"  {api.ret} (*{api.real_name})({api.get_args()}) = nullptr;")
+
+    def init_api(self, api):
+        self.lines.append(f"    if (is_intercepted) {{")
+        self.lines.append(f"      {api.real_name} = ({api.type})dlsym(RTLD_NEXT, \"{api.name}\");")
+        self.lines.append(f"    }} else {{")
+        self.lines.append(f"      {api.real_name} = ({api.type})dlsym(RTLD_DEFAULT, \"{api.name}\");")
+        self.lines.append(f"    }}")
+        self.lines.append(f"    if ({api.real_name} == nullptr) {{")
+        self.lines.append(f"      LOG(FATAL) << \"HERMES Adapter failed to map symbol: \"")
+        self.lines.append(f"      \"{api.name}\" << std::endl;")
+        self.lines.append(f"    }}")

adapter/adapter_generator/mpiio.py

@@ -0,0 +1,39 @@
+from adapter_generator.create_interceptor import Api,ApiClass
+
+apis = [
+    Api("int MPI_Init(int *argc, char ***argv)"),
+    Api("int MPI_Finalize(void)"),
+    Api("int MPI_Wait(MPI_Request *req, MPI_Status *status)"),
+    Api("int MPI_Waitall(int count, MPI_Request *req, MPI_Status *status)"),
+    Api("int MPI_File_open(MPI_Comm comm, const char *filename, int amode, MPI_Info info, MPI_File *fh)"),
+    Api("int MPI_File_close(MPI_File *fh)"),
+    Api("int MPI_File_seek_shared(MPI_File fh, MPI_Offset offset, int whence)"),
+    Api("int MPI_File_seek(MPI_File fh, MPI_Offset offset, int whence)"),
+    Api("int MPI_File_get_position(MPI_File fh, MPI_Offset *offset)"),
+    Api("int MPI_File_read_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"),
+    Api("int MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"),
+    Api("int MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"),
+    Api("int MPI_File_read(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"),
+    Api("int MPI_File_read_ordered(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"),
+    Api("int MPI_File_read_shared(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"),
+    Api("int MPI_File_write_all(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"),
+    Api("int MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"),
+    Api("int MPI_File_write_at(MPI_File fh, MPI_Offset offset, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"),
+    Api("int MPI_File_write(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"),
+    Api("int MPI_File_write_ordered(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"),
+    Api("int MPI_File_write_shared(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"),
+    Api("int MPI_File_iread_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)"),
+    Api("int MPI_File_iread(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)"),
+    Api("int MPI_File_iread_shared(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)"),
+    Api("int MPI_File_iwrite_at(MPI_File fh, MPI_Offset offset, const void *buf, int count, MPI_Datatype datatype, MPI_Request *request)"),
+    Api("int MPI_File_iwrite(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Request *request)"),
+    Api("int MPI_File_iwrite_shared(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Request *request)"),
+    Api("int MPI_File_sync(MPI_File fh)"),
+]
+
+includes = [
+    "<mpi.h>",
+    "<mpio.h>"
+]
+
+ApiClass("mpiio", apis, includes)

adapter/adapter_generator/posix.py

@@ -0,0 +1,24 @@
+from adapter_generator.create_interceptor import Api,ApiClass
+
+apis = [
+    Api("int MPI_Init(int *argc, char ***argv)"),
+    Api("int MPI_Finalize(void)"),
+    Api("int open(const char *path, int flags, ...)"),
+    Api("int open64(const char *path, int flags, ...)"),
+    Api("int __open_2(const char *path, int oflag)"),
+    Api("int creat(const char *path, mode_t mode)"),
+    Api("int creat64(const char *path, mode_t mode)"),
+    Api("ssize_t read(int fd, void *buf, size_t count)"),
+    Api("ssize_t write(int fd, const void *buf, size_t count)"),
+    Api("ssize_t pread(int fd, void *buf, size_t count, off_t offset)"),
+    Api("ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset)"),
+    Api("ssize_t pread64(int fd, void *buf, size_t count, off64_t offset)"),
+    Api("ssize_t pwrite64(int fd, const void *buf, size_t count, off64_t offset)"),
+    Api("off_t lseek(int fd, off_t offset, int whence)"),
+    Api("off64_t lseek64(int fd, off64_t offset, int whence)"),
+    Api("int __fxstat(int version, int fd, struct stat *buf)"),
+    Api("int fsync(int fd)"),
+    Api("int close(int fd)"),
+]
+
+ApiClass("posix", apis, [])

adapter/adapter_generator/stdio.py

@@ -0,0 +1,38 @@
+from adapter_generator.create_interceptor import Api,ApiClass
+
+apis = [
+    Api("int MPI_Init(int *argc, char ***argv)"),
+    Api("int MPI_Finalize(void)"),
+    Api("FILE *fopen(const char *path, const char *mode)"),
+    Api("FILE *fopen64(const char *path, const char *mode)"),
+    Api("FILE *fdopen(int fd, const char *mode)"),
+    Api("FILE *freopen(const char *path, const char *mode, FILE *stream)"),
+    Api("FILE *freopen64(const char *path, const char *mode, FILE *stream)"),
+    Api("int fflush(FILE *fp)"),
+    Api("int fclose(FILE *fp)"),
+    Api("size_t fwrite(const void *ptr, size_t size, size_t nmemb, FILE *fp)"),
+    Api("int fputc(int c, FILE *fp)"),
+    Api("int fgetpos(FILE *fp, fpos_t *pos)"),
+    Api("int fgetpos64(FILE *fp, fpos64_t *pos)"),
+    Api("int putc(int c, FILE *fp)"),
+    Api("int putw(int w, FILE *fp)"),
+    Api("int fputs(const char *s, FILE *stream)"),
+    Api("size_t fread(void *ptr, size_t size, size_t nmemb, FILE *stream)"),
+    Api("int fgetc(FILE *stream)"),
+    Api("int getc(FILE *stream)"),
+    Api("int getw(FILE *stream)"),
+    Api("char *fgets(char *s, int size, FILE *stream)"),
+    Api("void rewind(FILE *stream)"),
+    Api("int fseek(FILE *stream, long offset, int whence)"),
+    Api("int fseeko(FILE *stream, off_t offset, int whence)"),
+    Api("int fseeko64(FILE *stream, off64_t offset, int whence)"),
+    Api("int fsetpos(FILE *stream, const fpos_t *pos)"),
+    Api("int fsetpos64(FILE *stream, const fpos64_t *pos)"),
+    Api("long int ftell(FILE *fp)"),
+]
+
+includes = [
+    "\"cstdio.h\""
+]
+
+ApiClass("stdio", apis, includes)