From 1d0b3d788691d20cb7c0b7a57f5c0af412b2b40a Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 18 Oct 2022 02:08:13 -0500 Subject: [PATCH 1/8] Preload test --- .../adapter_generator/create_interceptor.py | 51 +++++++++++++++---- adapter/adapter_generator/posix.py | 5 +- adapter/interceptor.h | 5 +- adapter/posix/CMakeLists.txt | 2 +- adapter/posix/posix.cc | 19 +++---- test/CMakeLists.txt | 2 + test/preload_test.cc | 13 +++++ 7 files changed, 74 insertions(+), 23 deletions(-) create mode 100644 test/preload_test.cc diff --git a/adapter/adapter_generator/adapter_generator/create_interceptor.py b/adapter/adapter_generator/adapter_generator/create_interceptor.py index 4188cf3b2..511836972 100644 --- a/adapter/adapter_generator/adapter_generator/create_interceptor.py +++ b/adapter/adapter_generator/adapter_generator/create_interceptor.py @@ -15,6 +15,7 @@ class Api: def __init__(self, api_str): + self.api_str = api_str self.decompose_prototype(api_str) def _is_text(self, tok): @@ -77,8 +78,41 @@ def pass_args(self): return ", ".join(args) class ApiClass: - def __init__(self, namespace, apis, includes, path=None, do_save=True): + def __init__(self, namespace, apis, includes, dir=None, + create_h=True, create_cc=False): self.apis = apis + h_file = None + cc_file = None + # Ensure that directory is set if create_cc or create_h is true + if create_h or create_cc: + if dir is None: + dir = os.path.dirname(os.getcwd()) + os.path.join(ns_dir, namespace) + if dir is not None: + h_file = os.path.join(dir, "real_api.h") + if dir is not None: + cc_file = os.path.join(dir, f"{namespace}.cc") + self._CreateH(namespace, h_file) + self._CreateCC(namespace, cc_file) + + def _CreateCC(self, namespace, path): + self.lines.append(preamble) + self.lines.append("") + self.lines.append(f"bool {namespace}_is_intercepted = true") + self.lines.append(f"#include \"real_api.h\"") + self.lines.append("") + for api in self.apis: + self.lines.append(f"{api.api_str} {{") + self.lines.append(f" auto real_api = Singleton::GetInstance();") + self.lines.append(f" // auto fs_api = Singleton::GetInstance();") + self.lines.append(f" return real_api->{api.name}({api.pass_args()});") + self.lines.append(f"}}") + self.lines.append("") + + text = "\n".join(self.lines) + self.save(path, text) + + def _CreateH(self, namespace, path, do_save): self.lines = [] self.lines.append(preamble) @@ -115,20 +149,17 @@ def __init__(self, namespace, apis, includes, path=None, do_save=True): self.lines.append("") self.lines.append(f"#endif // HERMES_ADAPTER_{namespace.upper()}_H") self.lines.append("") - self.text = "\n".join(self.lines) - if do_save: - self.save(path, namespace) - else: - print(self.text) + text = "\n".join(self.lines) + self.save(path, text) - def save(self, path, namespace): + def save(self, path, text): if path is None: - ns_dir = os.path.dirname(os.getcwd()) - path = os.path.join(ns_dir, namespace, f"real_api.h") + print(text) + return with open(path, "w") as fp: - fp.write(self.text) + fp.write(text) def add_intercept_api(self, api): self.lines.append(f" typedef {api.ret} (*{api.type})({api.get_args()});") diff --git a/adapter/adapter_generator/posix.py b/adapter/adapter_generator/posix.py index 391ebc36d..17a1d3e16 100644 --- a/adapter/adapter_generator/posix.py +++ b/adapter/adapter_generator/posix.py @@ -21,4 +21,7 @@ Api("int close(int fd)"), ] -ApiClass("posix", apis, []) \ No newline at end of file +ApiClass("posix", apis, [], + dir="/home/lukemartinlogan/Documents/Projects/PhD/hermes/test", + create_h=True, + create_cc=True) \ No newline at end of file diff --git a/adapter/interceptor.h b/adapter/interceptor.h index bb7e12c8b..98de4a513 100644 --- a/adapter/interceptor.h +++ b/adapter/interceptor.h @@ -58,6 +58,7 @@ inline std::vector StringSplit(const char* str, char delimiter) { inline std::string GetFilenameFromFP(FILE* fh) { char proclnk[kMaxPathLen]; char filename[kMaxPathLen]; + if (fh == nullptr) return ""; int fno = fileno(fh); snprintf(proclnk, kMaxPathLen, "/proc/self/fd/%d", fno); size_t r = readlink(proclnk, filename, kMaxPathLen); @@ -67,10 +68,10 @@ inline std::string GetFilenameFromFP(FILE* fh) { inline std::string GetFilenameFromFD(int fd) { char proclnk[kMaxPathLen]; char filename[kMaxPathLen]; + if (fd == -1) return ""; snprintf(proclnk, kMaxPathLen, "/proc/self/fd/%d", fd); size_t r = readlink(proclnk, filename, kMaxPathLen); - filename[r] = '\0'; - return filename; + return std::string(filename, r); } /** diff --git a/adapter/posix/CMakeLists.txt b/adapter/posix/CMakeLists.txt index edc46e87c..bb6778526 100644 --- a/adapter/posix/CMakeLists.txt +++ b/adapter/posix/CMakeLists.txt @@ -14,7 +14,7 @@ set(POSIX_ADAPTER_PUBLIC_HEADER # Add library hermes_posix_backend add_library(hermes_posix_backend ${CMAKE_CURRENT_SOURCE_DIR}/fs_api.cc) add_dependencies(hermes_posix_backend hermes) -target_link_libraries(hermes_posix_backend hermes MPI::MPI_CXX glog::glog stdc++fs dl) +target_link_libraries(hermes_posix_backend stdc++ hermes MPI::MPI_CXX glog::glog stdc++fs dl) # Add library hermes_posix add_library(hermes_posix SHARED ${POSIX_ADAPTER_PUBLIC_HEADER} ${POSIX_ADAPTER_SRC}) diff --git a/adapter/posix/posix.cc b/adapter/posix/posix.cc index 94cb71e6a..8982eda49 100644 --- a/adapter/posix/posix.cc +++ b/adapter/posix/posix.cc @@ -102,9 +102,9 @@ int HERMES_DECL(open64)(const char *path, int flags, ...) { mode = va_arg(arg, int); va_end(arg); } + LOG(INFO) << "Intercept open64 for filename: " << path + << " and mode: " << flags << " is tracked." << std::endl; if (hermes::adapter::IsTracked(path)) { - LOG(INFO) << "Intercept open for filename: " << path - << " and mode: " << flags << " is tracked." << std::endl; AdapterStat stat; stat.flags = flags; stat.st_mode = mode; @@ -119,9 +119,9 @@ int HERMES_DECL(open64)(const char *path, int flags, ...) { int HERMES_DECL(__open_2)(const char *path, int oflag) { auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); + LOG(INFO) << "Intercept __open_2 for filename: " << path + << " and mode: " << oflag << " is tracked." << std::endl; if (hermes::adapter::IsTracked(path)) { - LOG(INFO) << "Intercept __open_2 for filename: " << path - << " and mode: " << oflag << " is tracked." << std::endl; AdapterStat stat; stat.flags = oflag; stat.st_mode = 0; @@ -134,9 +134,9 @@ int HERMES_DECL(creat)(const char *path, mode_t mode) { std::string path_str(path); auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); + LOG(INFO) << "Intercept creat for filename: " << path + << " and mode: " << mode << " is tracked." << std::endl; if (hermes::adapter::IsTracked(path)) { - LOG(INFO) << "Intercept creat for filename: " << path - << " and mode: " << mode << " is tracked." << std::endl; AdapterStat stat; stat.flags = O_CREAT; stat.st_mode = mode; @@ -149,9 +149,9 @@ int HERMES_DECL(creat64)(const char *path, mode_t mode) { std::string path_str(path); auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); + LOG(INFO) << "Intercept creat64 for filename: " << path + << " and mode: " << mode << " is tracked." << std::endl; if (hermes::adapter::IsTracked(path)) { - LOG(INFO) << "Intercept creat64 for filename: " << path - << " and mode: " << mode << " is tracked." << std::endl; AdapterStat stat; stat.flags = O_CREAT; stat.st_mode = mode; @@ -323,10 +323,11 @@ int HERMES_DECL(close)(int fd) { auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fd)) { - File f; f.fd_ = fd; fs_api->_InitFile(f); LOG(INFO) << "Intercept close(" << std::to_string(fd) << ")"; DLOG(INFO) << " -> " << hermes::adapter::GetFilenameFromFD(fd); LOG(INFO) << std::endl; + + File f; f.fd_ = fd; fs_api->_InitFile(f); return fs_api->Close(f, stat_exists); } return real_api->close(fd); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index df1ea49fc..6e588b1fa 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -164,3 +164,5 @@ if(HERMES_INSTALL_TESTS) ) endforeach() endif() + +add_executable(preload_test preload_test.cc) \ No newline at end of file diff --git a/test/preload_test.cc b/test/preload_test.cc new file mode 100644 index 000000000..794634e4f --- /dev/null +++ b/test/preload_test.cc @@ -0,0 +1,13 @@ +// +// Created by lukemartinlogan on 10/17/22. +// + +#include +#include +#include + +int main() { + printf("HERE?\n"); + int fd = open("hello.txt", O_WRONLY); + close(fd); +} From 91ef1cfd20a4f669df318613bbca5792908c9dec Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 18 Oct 2022 03:37:40 -0500 Subject: [PATCH 2/8] Add cc adapter generator, begin REQUIRE for real_api --- .../adapter_generator/create_interceptor.py | 151 ++++++++++-------- adapter/adapter_generator/posix.py | 7 +- adapter/interceptor.h | 4 +- adapter/mapper/mapper_factory.h | 2 +- adapter/mpiio/fs_api.h | 2 +- adapter/mpiio/mpiio.cc | 2 +- adapter/posix/CMakeLists.txt | 2 +- adapter/posix/fs_api.h | 2 +- adapter/posix/posix.cc | 8 +- adapter/pubsub/CMakeLists.txt | 3 +- adapter/pubsub/pubsub.cc | 12 +- adapter/pubsub/pubsub.h | 2 +- adapter/stdio/fs_api.h | 2 +- adapter/stdio/stdio.cc | 2 +- .../pubsub/pubsub_end_to_end_test_sync.cc | 2 +- adapter/test/pubsub/pubsub_metadata_test.cc | 4 +- {adapter => src}/singleton.h | 2 +- test/CMakeLists.txt | 2 - 18 files changed, 119 insertions(+), 92 deletions(-) rename {adapter => src}/singleton.h (98%) diff --git a/adapter/adapter_generator/adapter_generator/create_interceptor.py b/adapter/adapter_generator/adapter_generator/create_interceptor.py index 511836972..85657b02f 100644 --- a/adapter/adapter_generator/adapter_generator/create_interceptor.py +++ b/adapter/adapter_generator/adapter_generator/create_interceptor.py @@ -74,7 +74,7 @@ def get_args(self): def pass_args(self): if self.var_defs is None: return "" - args = [arg[-1] for arg in self.var_defs] + args = [arg[-1] for arg in self.var_defs if arg[0] != ''] return ", ".join(args) class ApiClass: @@ -87,91 +87,116 @@ def __init__(self, namespace, apis, includes, dir=None, if create_h or create_cc: if dir is None: dir = os.path.dirname(os.getcwd()) - os.path.join(ns_dir, namespace) - if dir is not None: + dir = os.path.join(dir, namespace) + if dir is not None and create_h: h_file = os.path.join(dir, "real_api.h") - if dir is not None: + print(f"Will create header {h_file}") + if dir is not None and create_cc: cc_file = os.path.join(dir, f"{namespace}.cc") - self._CreateH(namespace, h_file) + print(f"Will create cpp file {h_file}") + self.cc_lines = [] + self.h_lines = [] + self._CreateH(namespace, includes, h_file) self._CreateCC(namespace, cc_file) def _CreateCC(self, namespace, path): - self.lines.append(preamble) - self.lines.append("") - self.lines.append(f"bool {namespace}_is_intercepted = true") - self.lines.append(f"#include \"real_api.h\"") - self.lines.append("") - for api in self.apis: - self.lines.append(f"{api.api_str} {{") - self.lines.append(f" auto real_api = Singleton::GetInstance();") - self.lines.append(f" // auto fs_api = Singleton::GetInstance();") - self.lines.append(f" return real_api->{api.name}({api.pass_args()});") - self.lines.append(f"}}") - self.lines.append("") - - text = "\n".join(self.lines) - self.save(path, text) + self.cc_lines.append(preamble) + self.cc_lines.append("") - def _CreateH(self, namespace, path, do_save): - self.lines = [] + # Includes + self.cc_lines.append(f"bool {namespace}_intercepted = true;") + self.cc_lines.append(f"#include \"real_api.h\"") + self.cc_lines.append(f"#include \"singleton.h\"") + self.cc_lines.append("") - self.lines.append(preamble) - self.lines.append("") - self.lines.append(f"#ifndef HERMES_ADAPTER_{namespace.upper()}_H") - self.lines.append(f"#define HERMES_ADAPTER_{namespace.upper()}_H") + # Namespace simplification + self.cc_lines.append(f"using hermes::adapter::{namespace}::API;") + self.cc_lines.append(f"using hermes::Singleton;") + self.cc_lines.append("") + + # Intercept function + for api in self.apis: + self.cc_lines.append(f"{api.api_str} {{") + self.cc_lines.append(f" auto real_api = Singleton::GetInstance();") + self.cc_lines.append(f" REQUIRE_API({api.name});") + self.cc_lines.append(f" // auto fs_api = ") + self.cc_lines.append(f" return real_api->{api.name}({api.pass_args()});") + self.cc_lines.append(f"}}") + self.cc_lines.append("") + + text = "\n".join(self.cc_lines) + self.save(path, text) - self.lines.append("#include ") - self.lines.append("#include ") - self.lines.append("#include ") - self.lines.append("#include ") - self.lines.append("#include \"interceptor.h\"") - self.lines.append("#include \"filesystem/filesystem.h\"") + def _CreateH(self, namespace, includes, path): + self.h_lines.append(preamble) + self.h_lines.append("") + self.h_lines.append(f"#ifndef HERMES_ADAPTER_{namespace.upper()}_H") + self.h_lines.append(f"#define HERMES_ADAPTER_{namespace.upper()}_H") + + # Include files + self.h_lines.append("#include ") + self.h_lines.append("#include ") + self.h_lines.append("#include ") + self.h_lines.append("#include ") for include in includes: - self.lines.append(f"#include {include}") - self.lines.append("") + self.h_lines.append(f"#include {include}") + self.h_lines.append("") + + # Require API macro + self.require_api() + self.h_lines.append("") - self.lines.append(f"namespace hermes::adapter::{namespace} {{") - self.lines.append(f"") - self.lines.append(f"class API {{") + # Create the class definition + self.h_lines.append(f"namespace hermes::adapter::{namespace} {{") + self.h_lines.append(f"") + self.h_lines.append(f"class API {{") - self.lines.append(f" public:") + # Create the typedefs + self.h_lines.append(f" public:") for api in self.apis: self.add_intercept_api(api) - self.lines.append(f" API() {{") - self.lines.append(f" void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, \"{namespace}_intercepted\");") + # Create the symbol mapper + self.h_lines.append(f" API() {{") + self.h_lines.append(f" void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, \"{namespace}_intercepted\");") for api in self.apis: self.init_api(api) - self.lines.append(f" }}") - self.lines.append(f"}};") - self.lines.append(f"}} // namespace hermes::adapter::{namespace}") + self.h_lines.append(f" }}") - self.lines.append("") - self.lines.append(f"#endif // HERMES_ADAPTER_{namespace.upper()}_H") - self.lines.append("") + # Create the symbol require function + self.h_lines.append("") - text = "\n".join(self.lines) + # End the class, namespace, and header guard + self.h_lines.append(f"}};") + self.h_lines.append(f"}} // namespace hermes::adapter::{namespace}") + self.h_lines.append("") + self.h_lines.append(f"#endif // HERMES_ADAPTER_{namespace.upper()}_H") + self.h_lines.append("") + + text = "\n".join(self.h_lines) self.save(path, text) + def require_api(self): + self.h_lines.append(f"#define REQUIRE_API(api_name) \\") + self.h_lines.append(f" if (real_api->api_name == nullptr) {{ \\") + self.h_lines.append(f" LOG(FATAL) << \"HERMES Adapter failed to map symbol: \" \\") + self.h_lines.append(f" #api_name << std::endl; \\") + self.h_lines.append(f" exit(1);") + + def add_intercept_api(self, api): + self.h_lines.append(f" typedef {api.ret} (*{api.type})({api.get_args()});") + self.h_lines.append(f" {api.ret} (*{api.real_name})({api.get_args()}) = nullptr;") + + def init_api(self, api): + self.h_lines.append(f" if (is_intercepted) {{") + self.h_lines.append(f" {api.real_name} = ({api.type})dlsym(RTLD_NEXT, \"{api.name}\");") + self.h_lines.append(f" }} else {{") + self.h_lines.append(f" {api.real_name} = ({api.type})dlsym(RTLD_DEFAULT, \"{api.name}\");") + self.h_lines.append(f" }}") def save(self, path, text): if path is None: print(text) return with open(path, "w") as fp: - fp.write(text) - - def add_intercept_api(self, api): - self.lines.append(f" typedef {api.ret} (*{api.type})({api.get_args()});") - self.lines.append(f" {api.ret} (*{api.real_name})({api.get_args()}) = nullptr;") - - def init_api(self, api): - self.lines.append(f" if (is_intercepted) {{") - self.lines.append(f" {api.real_name} = ({api.type})dlsym(RTLD_NEXT, \"{api.name}\");") - self.lines.append(f" }} else {{") - self.lines.append(f" {api.real_name} = ({api.type})dlsym(RTLD_DEFAULT, \"{api.name}\");") - self.lines.append(f" }}") - self.lines.append(f" if ({api.real_name} == nullptr) {{") - self.lines.append(f" LOG(FATAL) << \"HERMES Adapter failed to map symbol: \"") - self.lines.append(f" \"{api.name}\" << std::endl;") - self.lines.append(f" }}") \ No newline at end of file + fp.write(text) \ No newline at end of file diff --git a/adapter/adapter_generator/posix.py b/adapter/adapter_generator/posix.py index 17a1d3e16..ff1afc976 100644 --- a/adapter/adapter_generator/posix.py +++ b/adapter/adapter_generator/posix.py @@ -21,7 +21,12 @@ Api("int close(int fd)"), ] -ApiClass("posix", apis, [], +includes = [ + "", + "" +] + +ApiClass("posix", apis, includes, dir="/home/lukemartinlogan/Documents/Projects/PhD/hermes/test", create_h=True, create_cc=True) \ No newline at end of file diff --git a/adapter/interceptor.h b/adapter/interceptor.h index 98de4a513..2e571c56a 100644 --- a/adapter/interceptor.h +++ b/adapter/interceptor.h @@ -32,10 +32,10 @@ * Define Interceptor list for adapter. */ #define INTERCEPTOR_LIST \ - hermes::adapter::Singleton::GetInstance<>() + hermes::Singleton::GetInstance<>() #define HERMES_CONF \ - hermes::adapter::Singleton::GetInstance() + hermes::Singleton::GetInstance() // Path lengths are up to 4096 bytes const int kMaxPathLen = 4096; diff --git a/adapter/mapper/mapper_factory.h b/adapter/mapper/mapper_factory.h index de5c1864e..1a90120f7 100644 --- a/adapter/mapper/mapper_factory.h +++ b/adapter/mapper/mapper_factory.h @@ -30,7 +30,7 @@ class MapperFactory { AbstractMapper* Get(const MapperType &type) { switch (type) { case MapperType::BALANCED: { - return hermes::adapter::Singleton::GetInstance(); + return hermes::Singleton::GetInstance(); } default: { // TODO(hari): @error_handling Mapper not implemented diff --git a/adapter/mpiio/fs_api.h b/adapter/mpiio/fs_api.h index 18c0d2dda..bdc2dfd15 100644 --- a/adapter/mpiio/fs_api.h +++ b/adapter/mpiio/fs_api.h @@ -34,7 +34,7 @@ namespace hermes::adapter::mpiio { using hermes::adapter::fs::AdapterStat; using hermes::adapter::fs::File; -using hermes::adapter::Singleton; +using hermes::Singleton; using hermes::adapter::mpiio::API; using hermes::adapter::fs::IoOptions; using hermes::adapter::fs::IoStatus; diff --git a/adapter/mpiio/mpiio.cc b/adapter/mpiio/mpiio.cc index afed8452e..7b71d5662 100644 --- a/adapter/mpiio/mpiio.cc +++ b/adapter/mpiio/mpiio.cc @@ -35,7 +35,7 @@ using hermes::adapter::fs::File; using hermes::adapter::mpiio::API; using hermes::adapter::mpiio::MpiioFS; using hermes::adapter::mpiio::MpiioSeekModeConv; -using hermes::adapter::Singleton; +using hermes::Singleton; namespace hapi = hermes::api; namespace stdfs = std::experimental::filesystem; diff --git a/adapter/posix/CMakeLists.txt b/adapter/posix/CMakeLists.txt index bb6778526..edc46e87c 100644 --- a/adapter/posix/CMakeLists.txt +++ b/adapter/posix/CMakeLists.txt @@ -14,7 +14,7 @@ set(POSIX_ADAPTER_PUBLIC_HEADER # Add library hermes_posix_backend add_library(hermes_posix_backend ${CMAKE_CURRENT_SOURCE_DIR}/fs_api.cc) add_dependencies(hermes_posix_backend hermes) -target_link_libraries(hermes_posix_backend stdc++ hermes MPI::MPI_CXX glog::glog stdc++fs dl) +target_link_libraries(hermes_posix_backend hermes MPI::MPI_CXX glog::glog stdc++fs dl) # Add library hermes_posix add_library(hermes_posix SHARED ${POSIX_ADAPTER_PUBLIC_HEADER} ${POSIX_ADAPTER_SRC}) diff --git a/adapter/posix/fs_api.h b/adapter/posix/fs_api.h index 4e9ca7733..f7be4de4c 100644 --- a/adapter/posix/fs_api.h +++ b/adapter/posix/fs_api.h @@ -20,7 +20,7 @@ using hermes::adapter::fs::AdapterStat; using hermes::adapter::fs::File; -using hermes::adapter::Singleton; +using hermes::Singleton; using hermes::adapter::posix::API; using hermes::adapter::fs::IoOptions; using hermes::adapter::fs::IoStatus; diff --git a/adapter/posix/posix.cc b/adapter/posix/posix.cc index 8982eda49..cf3fca3bd 100644 --- a/adapter/posix/posix.cc +++ b/adapter/posix/posix.cc @@ -33,7 +33,7 @@ bool posix_intercepted = true; using hermes::adapter::WeaklyCanonical; using hermes::adapter::posix::API; using hermes::adapter::posix::PosixFS; -using hermes::adapter::Singleton; +using hermes::Singleton; using hermes::adapter::fs::MetadataManager; using hermes::adapter::fs::SeekMode; @@ -50,7 +50,7 @@ int HERMES_DECL(MPI_Init)(int *argc, char ***argv) { int status = real_api->MPI_Init(argc, argv); if (status == 0) { LOG(INFO) << "MPI Init intercepted." << std::endl; - auto mdm = hermes::adapter::Singleton::GetInstance(); + auto mdm = hermes::Singleton::GetInstance(); mdm->InitializeHermes(true); } return status; @@ -59,7 +59,7 @@ int HERMES_DECL(MPI_Init)(int *argc, char ***argv) { int HERMES_DECL(MPI_Finalize)(void) { auto real_api = Singleton::GetInstance(); LOG(INFO) << "MPI Finalize intercepted." << std::endl; - auto mdm = hermes::adapter::Singleton::GetInstance(); + auto mdm = hermes::Singleton::GetInstance(); mdm->FinalizeHermes(); int status = real_api->MPI_Finalize(); return status; @@ -275,7 +275,7 @@ int HERMES_DECL(__fxstat)(int version, int fd, struct stat *buf) { if (hermes::adapter::IsTracked(fd)) { File f; f.fd_ = fd; fs_api->_InitFile(f); LOG(INFO) << "Intercepted fstat." << std::endl; - auto mdm = hermes::adapter::Singleton::GetInstance(); + auto mdm = hermes::Singleton::GetInstance(); auto existing = mdm->Find(f); if (existing.second) { AdapterStat &astat = existing.first; diff --git a/adapter/pubsub/CMakeLists.txt b/adapter/pubsub/CMakeLists.txt index f59ba11f8..35a88f503 100644 --- a/adapter/pubsub/CMakeLists.txt +++ b/adapter/pubsub/CMakeLists.txt @@ -5,8 +5,7 @@ set(HERMES_PUBSUB_ADAPTER_DIR ${HERMES_ADAPTER_DIR}/pubsub) set(PUBSUB_ADAPTER_PUBLIC_HEADER pubsub.h) set(PUBSUB_ADAPTER_PRIVATE_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/metadata_manager.h ${CMAKE_CURRENT_SOURCE_DIR}/datastructures.h - ${HERMES_ADAPTER_DIR}/constants.h - ${HERMES_ADAPTER_DIR}/singleton.h) + ${HERMES_ADAPTER_DIR}/constants.h) set(PUBSUB_ADAPTER_SRC pubsub.cc metadata_manager.cc) add_library(hermes_pubsub SHARED ${PUBSUB_ADAPTER_PRIVATE_HEADER} ${PUBSUB_ADAPTER_PUBLIC_HEADER} ${PUBSUB_ADAPTER_SRC}) diff --git a/adapter/pubsub/pubsub.cc b/adapter/pubsub/pubsub.cc index 41ba2c73f..21c739fd6 100644 --- a/adapter/pubsub/pubsub.cc +++ b/adapter/pubsub/pubsub.cc @@ -27,7 +27,7 @@ hapi::Status hermes::pubsub::mpiInit(int argc, char **argv) { hapi::Status hermes::pubsub::connect(const std::string &config_file) { LOG(INFO) << "Connecting adapter" << std::endl; - auto mdm = hermes::adapter::Singleton + auto mdm = hermes::Singleton ::GetInstance(); try { mdm->InitializeHermes(config_file.c_str()); @@ -49,7 +49,7 @@ hapi::Status hermes::pubsub::connect() { hapi::Status hermes::pubsub::disconnect() { LOG(INFO) << "Disconnecting adapter" << std::endl; - auto mdm = hermes::adapter::Singleton + auto mdm = hermes::Singleton ::GetInstance(); try { mdm->FinalizeHermes(); @@ -63,7 +63,7 @@ hapi::Status hermes::pubsub::disconnect() { hapi::Status hermes::pubsub::attach(const std::string& topic) { LOG(INFO) << "Attaching to topic: " << topic << std::endl; hapi::Context ctx; - auto mdm = hermes::adapter::Singleton + auto mdm = hermes::Singleton ::GetInstance(); auto existing = mdm->Find(topic); if (!existing.second) { @@ -90,7 +90,7 @@ hapi::Status hermes::pubsub::attach(const std::string& topic) { hapi::Status hermes::pubsub::detach(const std::string& topic) { LOG(INFO) << "Detaching from topic: " << topic << std::endl; hapi::Context ctx; - auto mdm = hermes::adapter::Singleton + auto mdm = hermes::Singleton ::GetInstance(); auto existing = mdm->Find(topic); if (existing.second) { @@ -103,7 +103,7 @@ hapi::Status hermes::pubsub::detach(const std::string& topic) { hapi::Status hermes::pubsub::publish(const std::string& topic, const hapi::Blob& message) { LOG(INFO) << "Publish to : " << topic << std::endl; - auto mdm = hermes::adapter::Singleton + auto mdm = hermes::Singleton ::GetInstance(); auto metadata = mdm->Find(topic); @@ -138,7 +138,7 @@ std::pair hermes::pubsub::subscribe( LOG(INFO) << "Subscribe to : " << topic << std::endl; typedef std::pair SubscribeReturn; - auto mdm = hermes::adapter::Singleton + auto mdm = hermes::Singleton ::GetInstance(); auto metadata = mdm->Find(topic); diff --git a/adapter/pubsub/pubsub.h b/adapter/pubsub/pubsub.h index 70f86e2d4..c296e661f 100644 --- a/adapter/pubsub/pubsub.h +++ b/adapter/pubsub/pubsub.h @@ -31,7 +31,7 @@ #include #include "metadata_manager.h" #include "../constants.h" -#include "../singleton.h" +#include "singleton.h" namespace hermes::pubsub { diff --git a/adapter/stdio/fs_api.h b/adapter/stdio/fs_api.h index c3550342e..3dfe80cf6 100644 --- a/adapter/stdio/fs_api.h +++ b/adapter/stdio/fs_api.h @@ -23,7 +23,7 @@ using hermes::adapter::fs::AdapterStat; using hermes::adapter::fs::File; -using hermes::adapter::Singleton; +using hermes::Singleton; using hermes::adapter::stdio::API; using hermes::adapter::fs::IoOptions; using hermes::adapter::fs::IoStatus; diff --git a/adapter/stdio/stdio.cc b/adapter/stdio/stdio.cc index 022f130ad..e1235553b 100644 --- a/adapter/stdio/stdio.cc +++ b/adapter/stdio/stdio.cc @@ -23,7 +23,7 @@ bool stdio_intercepted = true; using hermes::adapter::WeaklyCanonical; using hermes::adapter::stdio::API; using hermes::adapter::stdio::StdioFS; -using hermes::adapter::Singleton; +using hermes::Singleton; using hermes::adapter::fs::MetadataManager; using hermes::adapter::fs::SeekMode; diff --git a/adapter/test/pubsub/pubsub_end_to_end_test_sync.cc b/adapter/test/pubsub/pubsub_end_to_end_test_sync.cc index 376e06e74..6e5839e08 100644 --- a/adapter/test/pubsub/pubsub_end_to_end_test_sync.cc +++ b/adapter/test/pubsub/pubsub_end_to_end_test_sync.cc @@ -46,7 +46,7 @@ int main(int argc, char **argv) { Assert(publish_ret.Succeeded()); } - auto hermes = hermes::adapter::Singleton + auto hermes = hermes::Singleton ::GetInstance()->GetHermes(); MPI_Comm comm = *(MPI_Comm*)hermes->GetAppCommunicator(); MPI_Barrier(comm); diff --git a/adapter/test/pubsub/pubsub_metadata_test.cc b/adapter/test/pubsub/pubsub_metadata_test.cc index 93a28af4b..051fc2ab2 100644 --- a/adapter/test/pubsub/pubsub_metadata_test.cc +++ b/adapter/test/pubsub/pubsub_metadata_test.cc @@ -11,11 +11,11 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include -#include +#include "singleton.h" #include "test_utils.h" int main() { - auto mdm = hermes::adapter::Singleton + auto mdm = hermes::Singleton ::GetInstance(false); ClientMetadata stat; struct timespec ts{}; diff --git a/adapter/singleton.h b/src/singleton.h similarity index 98% rename from adapter/singleton.h rename to src/singleton.h index 9ab2a1959..31e04420a 100644 --- a/adapter/singleton.h +++ b/src/singleton.h @@ -21,7 +21,7 @@ * Singleton::GetInstance() * @tparam T */ -namespace hermes::adapter { +namespace hermes { template class Singleton { public: diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6e588b1fa..df1ea49fc 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -164,5 +164,3 @@ if(HERMES_INSTALL_TESTS) ) endforeach() endif() - -add_executable(preload_test preload_test.cc) \ No newline at end of file From f52a197e0b6ac31a3782323ec96e2ec01f50457d Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 18 Oct 2022 04:01:07 -0500 Subject: [PATCH 3/8] Remove custom posix generator --- adapter/adapter_generator/posix.py | 5 +---- adapter/interceptor.h | 6 ++++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/adapter/adapter_generator/posix.py b/adapter/adapter_generator/posix.py index ff1afc976..f7a0233b1 100644 --- a/adapter/adapter_generator/posix.py +++ b/adapter/adapter_generator/posix.py @@ -26,7 +26,4 @@ "" ] -ApiClass("posix", apis, includes, - dir="/home/lukemartinlogan/Documents/Projects/PhD/hermes/test", - create_h=True, - create_cc=True) \ No newline at end of file +ApiClass("posix", apis, includes) \ No newline at end of file diff --git a/adapter/interceptor.h b/adapter/interceptor.h index 2e571c56a..6337fabbb 100644 --- a/adapter/interceptor.h +++ b/adapter/interceptor.h @@ -201,6 +201,12 @@ void OnExit(void); } \ } +#define REQUIRE_API(api_name) \ + if (real_api->api_name == nullptr) { \ + LOG(FATAL) << "HERMES Adapter failed to map symbol: " \ + #api_name << std::endl; \ + exit(1); + namespace hermes::adapter { /** * Loads the buffering paths for Hermes Adapter to exclude. It inserts the From 20d5b63237daa28af5e01b21182726788c8a6069 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 18 Oct 2022 10:10:14 -0500 Subject: [PATCH 4/8] IsTracked with bools --- adapter/interceptor.cc | 53 ++++++++++++++++++++++++++++++++++-------- adapter/interceptor.h | 5 ++-- adapter/posix/posix.cc | 3 ++- 3 files changed, 48 insertions(+), 13 deletions(-) diff --git a/adapter/interceptor.cc b/adapter/interceptor.cc index ca477c476..0cd67d527 100644 --- a/adapter/interceptor.cc +++ b/adapter/interceptor.cc @@ -73,7 +73,8 @@ void PopulateBufferingPath() { populated = true; } -bool IsTracked(const std::string& path) { +bool IsTracked(const std::string& path, bool log) { + if (log) { LOG(INFO) << "IsTracked path? " << path << std::endl; } if (hermes::adapter::exit) { return false; } @@ -82,7 +83,11 @@ bool IsTracked(const std::string& path) { std::string abs_path = WeaklyCanonical(path).string(); for (const auto& pth : INTERCEPTOR_LIST->hermes_flush_exclusion) { - if (abs_path.find(pth) != std::string::npos) { + if (abs_path.rfind(pth) != std::string::npos) { + if (log) { + LOG(INFO) << "Path " << path << " is not tracked (hermes_flush_exclusion)" + << " because: " << pth << std::endl; + } return false; } } @@ -92,20 +97,31 @@ bool IsTracked(const std::string& path) { } for (const auto& pth : INTERCEPTOR_LIST->hermes_paths_inclusion) { - if (abs_path.find(pth) != std::string::npos) { + if (abs_path.rfind(pth) != std::string::npos) { + if (log) { + LOG(INFO) << "Path " << path << "is tracked (hermes_paths_inclusion)" + << " because: " << pth << std::endl; + } return true; } } for (auto &pth : kPathExclusions) { - if (abs_path.find(pth) != std::string::npos) { + if (abs_path.rfind(pth) != std::string::npos) { + if (log) { + LOG(INFO) << "Path " << path << " is not tracked (kPathExclusions)" + << " because: " << pth << std::endl; + } return false; } } for (const auto& pth : INTERCEPTOR_LIST->hermes_paths_exclusion) { - if (abs_path.find(pth) != std::string::npos || - pth.find(abs_path) != std::string::npos) { + if (abs_path.rfind(pth) != std::string::npos) { + if (log) { + LOG(INFO) << "Path " << path << " is not tracked (hermes_paths_exclusion)" + << " because: " << pth << std::endl; + } return false; } } @@ -114,10 +130,18 @@ bool IsTracked(const std::string& path) { auto buffer_mode = INTERCEPTOR_LIST->adapter_mode; if (buffer_mode == AdapterMode::kBypass) { if (list->adapter_paths.empty()) { + if (log) { + LOG(INFO) << "Path " << path << " is not tracked (kBypass)" + << " because adapter_paths empty " << std::endl; + } return false; } else { for (const auto& pth : list->adapter_paths) { if (path.find(pth) == 0) { + if (log) { + LOG(INFO) << "Path " << path << " is not tracked (kBypass)" + << " because: " << pth << std::endl; + } return false; } } @@ -128,16 +152,25 @@ bool IsTracked(const std::string& path) { } } -bool IsTracked(FILE* fh) { +bool IsTracked(FILE* fh, bool log) { + if (log) { LOG(INFO) << "IsTracked fh?" << std::endl; } if (hermes::adapter::exit) return false; atexit(OnExit); - return IsTracked(GetFilenameFromFP(fh)); + std::string path = GetFilenameFromFP(fh); + if (path == "") return false; + return IsTracked(path, log); } -bool IsTracked(int fd) { +bool IsTracked(int fd, bool log) { + if (log) { LOG(INFO) << "IsTracked fd?" << std::endl; } if (hermes::adapter::exit) return false; atexit(OnExit); - return IsTracked(GetFilenameFromFD(fd)); + std::string path = GetFilenameFromFD(fd); + if (path == "") { + if (log) { LOG(INFO) << fd << "is not tracked" << std::endl; } + return false; + } + return IsTracked(path, log); } void OnExit(void) { hermes::adapter::exit = true; } diff --git a/adapter/interceptor.h b/adapter/interceptor.h index 6337fabbb..e99dbf355 100644 --- a/adapter/interceptor.h +++ b/adapter/interceptor.h @@ -223,7 +223,7 @@ void PopulateBufferingPath(); * @return true, if file should be tracked. * false, if file should not be intercepted. */ -bool IsTracked(const std::string& path); +bool IsTracked(const std::string& path, bool log = false); /** * Check if fh should be tracked. In this method, first Convert fh to path. @@ -233,7 +233,8 @@ bool IsTracked(const std::string& path); * @return true, if file should be tracked. * false, if file should not be intercepted. */ -bool IsTracked(FILE* fh); +bool IsTracked(FILE* fh, bool log = false); +bool IsTracked(int fd, bool log = false); } // namespace hermes::adapter #else /** diff --git a/adapter/posix/posix.cc b/adapter/posix/posix.cc index cf3fca3bd..23bc53171 100644 --- a/adapter/posix/posix.cc +++ b/adapter/posix/posix.cc @@ -78,7 +78,8 @@ int HERMES_DECL(open)(const char *path, int flags, ...) { mode = va_arg(arg, int); va_end(arg); } - if (hermes::adapter::IsTracked(path)) { + bool do_log = std::string(path) == "../testfile"; + if (hermes::adapter::IsTracked(path, do_log)) { LOG(INFO) << "Intercept open for filename: " << path << " and mode: " << flags << " is tracked." << std::endl; AdapterStat stat; From b898fe152e6cbf1c82f2c909f17d76076cd01182 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 18 Oct 2022 12:31:42 -0500 Subject: [PATCH 5/8] POSIX adapter again intercepts correctly API calls --- .../adapter_generator/create_interceptor.py | 14 +- adapter/adapter_generator/posix.py | 5 +- adapter/interceptor.cc | 6 +- adapter/interceptor.h | 23 +- adapter/posix/CMakeLists.txt | 1 + adapter/posix/posix.cc | 22 +- adapter/posix/real_api.h | 198 ++++++------------ src/singleton.h | 2 +- test/CMakeLists.txt | 2 +- test/preload_test.cc | 13 -- 10 files changed, 113 insertions(+), 173 deletions(-) delete mode 100644 test/preload_test.cc diff --git a/adapter/adapter_generator/adapter_generator/create_interceptor.py b/adapter/adapter_generator/adapter_generator/create_interceptor.py index 85657b02f..518f3e380 100644 --- a/adapter/adapter_generator/adapter_generator/create_interceptor.py +++ b/adapter/adapter_generator/adapter_generator/create_interceptor.py @@ -146,15 +146,23 @@ def _CreateH(self, namespace, includes, path): self.require_api() self.h_lines.append("") + # Create typedefs + self.h_lines.append(f"extern \"C\" {{") + for api in self.apis: + self.add_typedef(api) + self.h_lines.append(f"}}") + self.h_lines.append(f"") + # Create the class definition self.h_lines.append(f"namespace hermes::adapter::{namespace} {{") self.h_lines.append(f"") self.h_lines.append(f"class API {{") - # Create the typedefs + # Create class function pointers self.h_lines.append(f" public:") for api in self.apis: self.add_intercept_api(api) + self.h_lines.append(f"") # Create the symbol mapper self.h_lines.append(f" API() {{") @@ -183,8 +191,10 @@ def require_api(self): self.h_lines.append(f" #api_name << std::endl; \\") self.h_lines.append(f" exit(1);") + def add_typedef(self, api): + self.h_lines.append(f"typedef {api.ret} (*{api.type})({api.get_args()});") + def add_intercept_api(self, api): - self.h_lines.append(f" typedef {api.ret} (*{api.type})({api.get_args()});") self.h_lines.append(f" {api.ret} (*{api.real_name})({api.get_args()}) = nullptr;") def init_api(self, api): diff --git a/adapter/adapter_generator/posix.py b/adapter/adapter_generator/posix.py index f7a0233b1..f6a0defaa 100644 --- a/adapter/adapter_generator/posix.py +++ b/adapter/adapter_generator/posix.py @@ -23,7 +23,10 @@ includes = [ "", - "" + "", + "\"interceptor.h\"", + "\"filesystem/filesystem.h\"", + "\"filesystem/metadata_manager.h\"" ] ApiClass("posix", apis, includes) \ No newline at end of file diff --git a/adapter/interceptor.cc b/adapter/interceptor.cc index 0cd67d527..41180f30b 100644 --- a/adapter/interceptor.cc +++ b/adapter/interceptor.cc @@ -85,7 +85,8 @@ bool IsTracked(const std::string& path, bool log) { for (const auto& pth : INTERCEPTOR_LIST->hermes_flush_exclusion) { if (abs_path.rfind(pth) != std::string::npos) { if (log) { - LOG(INFO) << "Path " << path << " is not tracked (hermes_flush_exclusion)" + LOG(INFO) << "Path " << path + << " is not tracked (hermes_flush_exclusion)" << " because: " << pth << std::endl; } return false; @@ -119,7 +120,8 @@ bool IsTracked(const std::string& path, bool log) { for (const auto& pth : INTERCEPTOR_LIST->hermes_paths_exclusion) { if (abs_path.rfind(pth) != std::string::npos) { if (log) { - LOG(INFO) << "Path " << path << " is not tracked (hermes_paths_exclusion)" + LOG(INFO) << "Path " << path + << " is not tracked (hermes_paths_exclusion)" << " because: " << pth << std::endl; } return false; diff --git a/adapter/interceptor.h b/adapter/interceptor.h index e99dbf355..8ede96e0c 100644 --- a/adapter/interceptor.h +++ b/adapter/interceptor.h @@ -55,23 +55,24 @@ inline std::vector StringSplit(const char* str, char delimiter) { } return v; } -inline std::string GetFilenameFromFP(FILE* fh) { - char proclnk[kMaxPathLen]; - char filename[kMaxPathLen]; - if (fh == nullptr) return ""; - int fno = fileno(fh); - snprintf(proclnk, kMaxPathLen, "/proc/self/fd/%d", fno); - size_t r = readlink(proclnk, filename, kMaxPathLen); - filename[r] = '\0'; - return filename; -} + inline std::string GetFilenameFromFD(int fd) { char proclnk[kMaxPathLen]; char filename[kMaxPathLen]; if (fd == -1) return ""; snprintf(proclnk, kMaxPathLen, "/proc/self/fd/%d", fd); size_t r = readlink(proclnk, filename, kMaxPathLen); - return std::string(filename, r); + if (r > 0) { + return std::string(filename, r); + } else { + return ""; + } +} + +inline std::string GetFilenameFromFP(FILE* fh) { + if (fh == nullptr) return ""; + int fno = fileno(fh); + return GetFilenameFromFD(fno); } /** diff --git a/adapter/posix/CMakeLists.txt b/adapter/posix/CMakeLists.txt index edc46e87c..56457400b 100644 --- a/adapter/posix/CMakeLists.txt +++ b/adapter/posix/CMakeLists.txt @@ -13,6 +13,7 @@ set(POSIX_ADAPTER_PUBLIC_HEADER # Add library hermes_posix_backend add_library(hermes_posix_backend ${CMAKE_CURRENT_SOURCE_DIR}/fs_api.cc) +target_compile_options(hermes_posix_backend PUBLIC -fPIC) add_dependencies(hermes_posix_backend hermes) target_link_libraries(hermes_posix_backend hermes MPI::MPI_CXX glog::glog stdc++fs dl) diff --git a/adapter/posix/posix.cc b/adapter/posix/posix.cc index 23bc53171..fbb1b7c01 100644 --- a/adapter/posix/posix.cc +++ b/adapter/posix/posix.cc @@ -41,6 +41,7 @@ namespace hapi = hermes::api; namespace stdfs = std::experimental::filesystem; using hermes::u8; +extern "C" { /** * MPI @@ -78,8 +79,7 @@ int HERMES_DECL(open)(const char *path, int flags, ...) { mode = va_arg(arg, int); va_end(arg); } - bool do_log = std::string(path) == "../testfile"; - if (hermes::adapter::IsTracked(path, do_log)) { + if (hermes::adapter::IsTracked(path)) { LOG(INFO) << "Intercept open for filename: " << path << " and mode: " << flags << " is tracked." << std::endl; AdapterStat stat; @@ -103,9 +103,9 @@ int HERMES_DECL(open64)(const char *path, int flags, ...) { mode = va_arg(arg, int); va_end(arg); } - LOG(INFO) << "Intercept open64 for filename: " << path - << " and mode: " << flags << " is tracked." << std::endl; if (hermes::adapter::IsTracked(path)) { + LOG(INFO) << "Intercept open64 for filename: " << path + << " and mode: " << flags << " is tracked." << std::endl; AdapterStat stat; stat.flags = flags; stat.st_mode = mode; @@ -120,9 +120,9 @@ int HERMES_DECL(open64)(const char *path, int flags, ...) { int HERMES_DECL(__open_2)(const char *path, int oflag) { auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); - LOG(INFO) << "Intercept __open_2 for filename: " << path - << " and mode: " << oflag << " is tracked." << std::endl; if (hermes::adapter::IsTracked(path)) { + LOG(INFO) << "Intercept __open_2 for filename: " << path + << " and mode: " << oflag << " is tracked." << std::endl; AdapterStat stat; stat.flags = oflag; stat.st_mode = 0; @@ -135,9 +135,9 @@ int HERMES_DECL(creat)(const char *path, mode_t mode) { std::string path_str(path); auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); - LOG(INFO) << "Intercept creat for filename: " << path - << " and mode: " << mode << " is tracked." << std::endl; if (hermes::adapter::IsTracked(path)) { + LOG(INFO) << "Intercept creat for filename: " << path + << " and mode: " << mode << " is tracked." << std::endl; AdapterStat stat; stat.flags = O_CREAT; stat.st_mode = mode; @@ -150,9 +150,9 @@ int HERMES_DECL(creat64)(const char *path, mode_t mode) { std::string path_str(path); auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); - LOG(INFO) << "Intercept creat64 for filename: " << path - << " and mode: " << mode << " is tracked." << std::endl; if (hermes::adapter::IsTracked(path)) { + LOG(INFO) << "Intercept creat64 for filename: " << path + << " and mode: " << mode << " is tracked." << std::endl; AdapterStat stat; stat.flags = O_CREAT; stat.st_mode = mode; @@ -333,3 +333,5 @@ int HERMES_DECL(close)(int fd) { } return real_api->close(fd); } + +} // extern C diff --git a/adapter/posix/real_api.h b/adapter/posix/real_api.h index ee8de2f30..ac2e64c3e 100644 --- a/adapter/posix/real_api.h +++ b/adapter/posix/real_api.h @@ -16,229 +16,163 @@ #include #include #include +#include +#include #include "interceptor.h" #include "filesystem/filesystem.h" #include "filesystem/metadata_manager.h" +#define REQUIRE_API(api_name) \ + if (real_api->api_name == nullptr) { \ + LOG(FATAL) << "HERMES Adapter failed to map symbol: " \ + #api_name << std::endl; \ + exit(1); + +extern "C" { +typedef int (*MPI_Init_t)(int * argc, char *** argv); +typedef int (*MPI_Finalize_t)( void); +typedef int (*open_t)(const char * path, int flags, ...); +typedef int (*open64_t)(const char * path, int flags, ...); +typedef int (*__open_2_t)(const char * path, int oflag); +typedef int (*creat_t)(const char * path, mode_t mode); +typedef int (*creat64_t)(const char * path, mode_t mode); +typedef ssize_t (*read_t)(int fd, void * buf, size_t count); +typedef ssize_t (*write_t)(int fd, const void * buf, size_t count); +typedef ssize_t (*pread_t)(int fd, void * buf, size_t count, off_t offset); +typedef ssize_t (*pwrite_t)(int fd, const void * buf, + size_t count, off_t offset); +typedef ssize_t (*pread64_t)(int fd, void * buf, size_t count, off64_t offset); +typedef ssize_t (*pwrite64_t)(int fd, const void * buf, + size_t count, off64_t offset); +typedef off_t (*lseek_t)(int fd, off_t offset, int whence); +typedef off64_t (*lseek64_t)(int fd, off64_t offset, int whence); +typedef int (*__fxstat_t)(int version, int fd, struct stat * buf); +typedef int (*fsync_t)(int fd); +typedef int (*close_t)(int fd); +} + namespace hermes::adapter::posix { class API { public: - typedef int(*open_t)(const char * path, int flags, ...); - int(*open)(const char * path, int flags, ...) = nullptr; - typedef int(*open64_t)(const char * path, int flags, ...); - int(*open64)(const char * path, int flags, ...) = nullptr; - typedef int(*__open_2_t)(const char * path, int oflag); - int(*__open_2)(const char * path, int oflag) = nullptr; - typedef int(*creat_t)(const char * path, mode_t mode); - int(*creat)(const char * path, mode_t mode) = nullptr; - typedef int(*creat64_t)(const char * path, mode_t mode); - int(*creat64)(const char * path, mode_t mode) = nullptr; - typedef ssize_t(*read_t)(int fd, void * buf, size_t count); - ssize_t(*read)(int fd, void * buf, size_t count) = nullptr; - typedef ssize_t(*write_t)(int fd, const void * buf, size_t count); - ssize_t(*write)(int fd, const void * buf, size_t count) = nullptr; - typedef ssize_t(*pread_t)(int fd, void * buf, size_t count, off_t offset); - ssize_t(*pread)(int fd, void * buf, size_t count, off_t offset) = nullptr; - typedef ssize_t(*pwrite_t)(int fd, const void * buf, - size_t count, off_t offset); - ssize_t(*pwrite)(int fd, const void * buf, + int (*MPI_Init)(int * argc, char *** argv) = nullptr; + int (*MPI_Finalize)( void) = nullptr; + int (*open)(const char * path, int flags, ...) = nullptr; + int (*open64)(const char * path, int flags, ...) = nullptr; + int (*__open_2)(const char * path, int oflag) = nullptr; + int (*creat)(const char * path, mode_t mode) = nullptr; + int (*creat64)(const char * path, mode_t mode) = nullptr; + ssize_t (*read)(int fd, void * buf, size_t count) = nullptr; + ssize_t (*write)(int fd, const void * buf, size_t count) = nullptr; + ssize_t (*pread)(int fd, void * buf, + size_t count, off_t offset) = nullptr; + ssize_t (*pwrite)(int fd, const void * buf, size_t count, off_t offset) = nullptr; - typedef ssize_t(*pread64_t)(int fd, void * buf, size_t count, off64_t offset); - ssize_t(*pread64)(int fd, void * buf, size_t count, off64_t offset) = nullptr; - typedef ssize_t(*pwrite64_t)(int fd, const void * buf, - size_t count, off64_t offset); - ssize_t(*pwrite64)(int fd, const void * buf, + ssize_t (*pread64)(int fd, void * buf, + size_t count, off64_t offset) = nullptr; + ssize_t (*pwrite64)(int fd, const void * buf, size_t count, off64_t offset) = nullptr; - typedef off_t(*lseek_t)(int fd, off_t offset, int whence); - off_t(*lseek)(int fd, off_t offset, int whence) = nullptr; - typedef off64_t(*lseek64_t)(int fd, off64_t offset, int whence); - off64_t(*lseek64)(int fd, off64_t offset, int whence) = nullptr; - typedef int(*__fxstat_t)(int version, int fd, struct stat * buf); - int(*__fxstat)(int version, int fd, struct stat * buf) = nullptr; - typedef int(*fsync_t)(int fd); - int(*fsync)(int fd) = nullptr; - typedef int(*fdatasync_t)(int fd); - int(*fdatasync)(int fd) = nullptr; - typedef int(*close_t)(int fd); - int(*close)(int fd) = nullptr; - typedef int(*MPI_Init_t)(int * argc, char *** argv); - int(*MPI_Init)(int * argc, char *** argv) = nullptr; - typedef int(*MPI_Finalize_t)(); - int(*MPI_Finalize)() = nullptr; + off_t (*lseek)(int fd, off_t offset, int whence) = nullptr; + off64_t (*lseek64)(int fd, off64_t offset, int whence) = nullptr; + int (*__fxstat)(int version, int fd, struct stat * buf) = nullptr; + int (*fsync)(int fd) = nullptr; + int (*close)(int fd) = nullptr; + API() { - void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, "posix_intercepted"); + void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, + "posix_intercepted"); + if (is_intercepted) { + MPI_Init = (MPI_Init_t)dlsym(RTLD_NEXT, "MPI_Init"); + } else { + MPI_Init = (MPI_Init_t)dlsym(RTLD_DEFAULT, "MPI_Init"); + } + if (is_intercepted) { + MPI_Finalize = (MPI_Finalize_t)dlsym( + RTLD_NEXT, "MPI_Finalize"); + } else { + MPI_Finalize = (MPI_Finalize_t)dlsym( + RTLD_DEFAULT, "MPI_Finalize"); + } if (is_intercepted) { open = (open_t)dlsym(RTLD_NEXT, "open"); } else { open = (open_t)dlsym(RTLD_DEFAULT, "open"); } - if (open == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "open" << std::endl; - } if (is_intercepted) { open64 = (open64_t)dlsym(RTLD_NEXT, "open64"); } else { open64 = (open64_t)dlsym(RTLD_DEFAULT, "open64"); } - if (open64 == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "open64" << std::endl; - } if (is_intercepted) { __open_2 = (__open_2_t)dlsym(RTLD_NEXT, "__open_2"); } else { __open_2 = (__open_2_t)dlsym(RTLD_DEFAULT, "__open_2"); } - if (__open_2 == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "__open_2" << std::endl; - } if (is_intercepted) { creat = (creat_t)dlsym(RTLD_NEXT, "creat"); } else { creat = (creat_t)dlsym(RTLD_DEFAULT, "creat"); } - if (creat == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "creat" << std::endl; - } if (is_intercepted) { creat64 = (creat64_t)dlsym(RTLD_NEXT, "creat64"); } else { creat64 = (creat64_t)dlsym(RTLD_DEFAULT, "creat64"); } - if (creat64 == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "creat64" << std::endl; - } if (is_intercepted) { read = (read_t)dlsym(RTLD_NEXT, "read"); } else { read = (read_t)dlsym(RTLD_DEFAULT, "read"); } - if (read == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "read" << std::endl; - } if (is_intercepted) { write = (write_t)dlsym(RTLD_NEXT, "write"); } else { write = (write_t)dlsym(RTLD_DEFAULT, "write"); } - if (write == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "write" << std::endl; - } if (is_intercepted) { pread = (pread_t)dlsym(RTLD_NEXT, "pread"); } else { pread = (pread_t)dlsym(RTLD_DEFAULT, "pread"); } - if (pread == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "pread" << std::endl; - } if (is_intercepted) { pwrite = (pwrite_t)dlsym(RTLD_NEXT, "pwrite"); } else { pwrite = (pwrite_t)dlsym(RTLD_DEFAULT, "pwrite"); } - if (pwrite == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "pwrite" << std::endl; - } if (is_intercepted) { pread64 = (pread64_t)dlsym(RTLD_NEXT, "pread64"); } else { pread64 = (pread64_t)dlsym(RTLD_DEFAULT, "pread64"); } - if (pread64 == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "pread64" << std::endl; - } if (is_intercepted) { pwrite64 = (pwrite64_t)dlsym(RTLD_NEXT, "pwrite64"); } else { pwrite64 = (pwrite64_t)dlsym(RTLD_DEFAULT, "pwrite64"); } - if (pwrite64 == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "pwrite64" << std::endl; - } if (is_intercepted) { lseek = (lseek_t)dlsym(RTLD_NEXT, "lseek"); } else { lseek = (lseek_t)dlsym(RTLD_DEFAULT, "lseek"); } - if (lseek == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "lseek" << std::endl; - } if (is_intercepted) { lseek64 = (lseek64_t)dlsym(RTLD_NEXT, "lseek64"); } else { lseek64 = (lseek64_t)dlsym(RTLD_DEFAULT, "lseek64"); } - if (lseek64 == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "lseek64" << std::endl; - } if (is_intercepted) { __fxstat = (__fxstat_t)dlsym(RTLD_NEXT, "__fxstat"); } else { __fxstat = (__fxstat_t)dlsym(RTLD_DEFAULT, "__fxstat"); } - if (__fxstat == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "__fxstat" << std::endl; - } if (is_intercepted) { fsync = (fsync_t)dlsym(RTLD_NEXT, "fsync"); } else { fsync = (fsync_t)dlsym(RTLD_DEFAULT, "fsync"); } - if (fsync == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fsync" << std::endl; - } - if (is_intercepted) { - fdatasync = (fdatasync_t)dlsym(RTLD_NEXT, "fdatasync"); - } else { - fdatasync = (fdatasync_t)dlsym(RTLD_DEFAULT, "fdatasync"); - } - if (fdatasync == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fdatasync" << std::endl; - } if (is_intercepted) { close = (close_t)dlsym(RTLD_NEXT, "close"); } else { close = (close_t)dlsym(RTLD_DEFAULT, "close"); } - if (close == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "close" << std::endl; - } - if (is_intercepted) { - MPI_Init = (MPI_Init_t)dlsym(RTLD_NEXT, "MPI_Init"); - } else { - MPI_Init = (MPI_Init_t)dlsym(RTLD_DEFAULT, "MPI_Init"); - } - if (MPI_Init == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_Init" << std::endl; - } - if (is_intercepted) { - MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_NEXT, "MPI_Finalize"); - } else { - MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_DEFAULT, "MPI_Finalize"); - } - if (MPI_Finalize == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_Finalize" << std::endl; - } } }; } // namespace hermes::adapter::posix diff --git a/src/singleton.h b/src/singleton.h index 31e04420a..4ff89afc4 100644 --- a/src/singleton.h +++ b/src/singleton.h @@ -58,5 +58,5 @@ class Singleton { template std::unique_ptr Singleton::instance = nullptr; -} // namespace hermes::adapter +} // namespace hermes #endif // HERMES_ADAPTER_SINGLETON_H diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index df1ea49fc..3a90c4740 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -163,4 +163,4 @@ if(HERMES_INSTALL_TESTS) RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} ) endforeach() -endif() +endif() \ No newline at end of file diff --git a/test/preload_test.cc b/test/preload_test.cc deleted file mode 100644 index 794634e4f..000000000 --- a/test/preload_test.cc +++ /dev/null @@ -1,13 +0,0 @@ -// -// Created by lukemartinlogan on 10/17/22. -// - -#include -#include -#include - -int main() { - printf("HERE?\n"); - int fd = open("hello.txt", O_WRONLY); - close(fd); -} From b594993bcc32e8e8127dfbef1656752e22f02c34 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 18 Oct 2022 12:48:43 -0500 Subject: [PATCH 6/8] Create unified lint script under ci. Ignore real_api files. --- .github/workflows/main.yml | 2 +- CMakeLists.txt | 3 +- .../adapter_generator/create_interceptor.py | 3 - adapter/mpiio/mpiio.cc | 128 +++++++---- adapter/stdio/stdio.cc | 207 +++++++++++++----- ci/lint.sh | 10 + 6 files changed, 252 insertions(+), 101 deletions(-) create mode 100644 ci/lint.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 701e11e7c..1e796464b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,7 +35,7 @@ jobs: uses: actions/setup-python@v1 - name: Run cpplint - run: pip install cpplint==1.5.4 && cpplint --recursive --exclude=src/stb_ds.h . + run: pip install cpplint==1.5.4 && bash ci/lint.sh `pwd` - name: Cache Spack packages uses: actions/cache@v2 diff --git a/CMakeLists.txt b/CMakeLists.txt index 49e13dfba..2506d6705 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -499,6 +499,5 @@ install( ) add_custom_target(lint - COMMAND cpplint --recursive --exclude=${CMAKE_CURRENT_SOURCE_DIR}/src/stb_ds.h - --exclude=${CMAKE_BINARY_DIR} ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/ci/lint.sh ${CMAKE_CURRENT_SOURCE_DIR} ) diff --git a/adapter/adapter_generator/adapter_generator/create_interceptor.py b/adapter/adapter_generator/adapter_generator/create_interceptor.py index 518f3e380..fc0251fef 100644 --- a/adapter/adapter_generator/adapter_generator/create_interceptor.py +++ b/adapter/adapter_generator/adapter_generator/create_interceptor.py @@ -171,9 +171,6 @@ def _CreateH(self, namespace, includes, path): self.init_api(api) self.h_lines.append(f" }}") - # Create the symbol require function - self.h_lines.append("") - # End the class, namespace, and header guard self.h_lines.append(f"}};") self.h_lines.append(f"}} // namespace hermes::adapter::{namespace}") diff --git a/adapter/mpiio/mpiio.cc b/adapter/mpiio/mpiio.cc index 7b71d5662..91c77f747 100644 --- a/adapter/mpiio/mpiio.cc +++ b/adapter/mpiio/mpiio.cc @@ -55,6 +55,8 @@ inline bool IsTracked(MPI_File *fh) { return exists; } +extern "C" { + /** * MPI */ @@ -115,7 +117,9 @@ int HERMES_DECL(MPI_File_close)(MPI_File *fh) { auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(fh)) { - File f; f.mpi_fh_ = *fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = *fh; + fs_api->_InitFile(f); return fs_api->Close(f, stat_exists); } return real_api->MPI_File_close(fh); @@ -126,7 +130,9 @@ int HERMES_DECL(MPI_File_seek)(MPI_File fh, MPI_Offset offset, int whence) { auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); return fs_api->Seek(f, stat_exists, offset, whence); } return real_api->MPI_File_seek(fh, offset, whence); @@ -140,7 +146,9 @@ int HERMES_DECL(MPI_File_seek_shared)(MPI_File fh, MPI_Offset offset, if (IsTracked(&fh)) { LOG(INFO) << "Intercept MPI_File_seek_shared offset:" << offset << " whence:" << whence << "." << std::endl; - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); return fs_api->SeekShared(f, stat_exists, offset, whence); } return real_api->MPI_File_seek_shared(fh, offset, whence); @@ -151,7 +159,9 @@ int HERMES_DECL(MPI_File_get_position)(MPI_File fh, MPI_Offset *offset) { auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); (*offset) = static_cast(fs_api->Tell(f, stat_exists)); return MPI_SUCCESS; } @@ -165,7 +175,9 @@ int HERMES_DECL(MPI_File_read_all)(MPI_File fh, void *buf, int count, auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { LOG(INFO) << "Intercept MPI_File_read_all." << std::endl; - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); return fs_api->ReadAll(f, stat_exists, buf, count, datatype, status); } return real_api->MPI_File_read_all(fh, buf, count, datatype, status); @@ -177,12 +189,14 @@ int HERMES_DECL(MPI_File_read_at_all)(MPI_File fh, MPI_Offset offset, void *buf, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); - return fs_api->ReadAll(f, stat_exists, buf, - offset, count, datatype, status); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); + return fs_api->ReadAll(f, stat_exists, buf, offset, count, datatype, + status); } - return real_api->MPI_File_read_at_all(fh, offset, buf, - count, datatype, status); + return real_api->MPI_File_read_at_all(fh, offset, buf, count, datatype, + status); } int HERMES_DECL(MPI_File_read_at)(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, @@ -191,7 +205,9 @@ int HERMES_DECL(MPI_File_read_at)(MPI_File fh, MPI_Offset offset, void *buf, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); return fs_api->Read(f, stat_exists, buf, offset, count, datatype, status); } return real_api->MPI_File_read_at(fh, offset, buf, count, datatype, status); @@ -202,7 +218,9 @@ int HERMES_DECL(MPI_File_read)(MPI_File fh, void *buf, int count, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); int ret = fs_api->Read(f, stat_exists, buf, count, datatype, status); if (stat_exists) return ret; } @@ -215,7 +233,9 @@ int HERMES_DECL(MPI_File_read_ordered)(MPI_File fh, void *buf, int count, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); return fs_api->ReadOrdered(f, stat_exists, buf, count, datatype, status); } return real_api->MPI_File_read_ordered(fh, buf, count, datatype, status); @@ -228,7 +248,9 @@ int HERMES_DECL(MPI_File_read_shared)(MPI_File fh, void *buf, int count, auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { LOG(INFO) << "Intercept MPI_File_read_shared." << std::endl; - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); return fs_api->Read(f, stat_exists, buf, count, datatype, status); } return real_api->MPI_File_read_shared(fh, buf, count, datatype, status); @@ -239,10 +261,12 @@ int HERMES_DECL(MPI_File_write_all)(MPI_File fh, const void *buf, int count, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - LOG(INFO) << "Intercept MPI_File_write_all." << std::endl; - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); - int ret = fs_api->WriteAll(f, stat_exists, buf, count, datatype, status); - if (stat_exists) return ret; + LOG(INFO) << "Intercept MPI_File_write_all." << std::endl; + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); + int ret = fs_api->WriteAll(f, stat_exists, buf, count, datatype, status); + if (stat_exists) return ret; } return real_api->MPI_File_write_all(fh, buf, count, datatype, status); } @@ -254,13 +278,15 @@ int HERMES_DECL(MPI_File_write_at_all)(MPI_File fh, MPI_Offset offset, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); - int ret = fs_api->WriteAll(f, stat_exists, buf, - offset, count, datatype, status); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); + int ret = + fs_api->WriteAll(f, stat_exists, buf, offset, count, datatype, status); if (stat_exists) return ret; } - return real_api->MPI_File_write_at_all(fh, offset, buf, count, - datatype, status); + return real_api->MPI_File_write_at_all(fh, offset, buf, count, datatype, + status); } int HERMES_DECL(MPI_File_write_at)(MPI_File fh, MPI_Offset offset, const void *buf, int count, @@ -270,7 +296,9 @@ int HERMES_DECL(MPI_File_write_at)(MPI_File fh, MPI_Offset offset, auto fs_api = Singleton::GetInstance(); LOG(INFO) << "In MPI_File_write_at" << std::endl; if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); return fs_api->Write(f, stat_exists, buf, offset, count, datatype, status); } return real_api->MPI_File_write_at(fh, offset, buf, count, datatype, status); @@ -282,7 +310,9 @@ int HERMES_DECL(MPI_File_write)(MPI_File fh, const void *buf, int count, auto fs_api = Singleton::GetInstance(); LOG(INFO) << "In MPI_File_write" << std::endl; if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); int ret = fs_api->Write(f, stat_exists, buf, count, datatype, status); if (stat_exists) return ret; } @@ -295,7 +325,9 @@ int HERMES_DECL(MPI_File_write_ordered)(MPI_File fh, const void *buf, int count, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); return fs_api->WriteOrdered(f, stat_exists, buf, count, datatype, status); } return real_api->MPI_File_write_ordered(fh, buf, count, datatype, status); @@ -308,7 +340,9 @@ int HERMES_DECL(MPI_File_write_shared)(MPI_File fh, const void *buf, int count, auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { // NOTE(llogan): originally WriteOrdered - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); return fs_api->WriteOrdered(f, stat_exists, buf, count, datatype, status); } return real_api->MPI_File_write_shared(fh, buf, count, datatype, status); @@ -324,12 +358,13 @@ int HERMES_DECL(MPI_File_iread_at)(MPI_File fh, MPI_Offset offset, void *buf, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); fs_api->ARead(f, stat_exists, buf, offset, count, datatype, request); return MPI_SUCCESS; } - return real_api->MPI_File_iread_at(fh, offset, buf, - count, datatype, request); + return real_api->MPI_File_iread_at(fh, offset, buf, count, datatype, request); } int HERMES_DECL(MPI_File_iread)(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request) { @@ -337,7 +372,9 @@ int HERMES_DECL(MPI_File_iread)(MPI_File fh, void *buf, int count, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); fs_api->ARead(f, stat_exists, buf, count, datatype, request); } return real_api->MPI_File_iread(fh, buf, count, datatype, request); @@ -349,7 +386,9 @@ int HERMES_DECL(MPI_File_iread_shared)(MPI_File fh, void *buf, int count, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); fs_api->ARead(f, stat_exists, buf, count, datatype, request); return MPI_SUCCESS; } @@ -363,12 +402,14 @@ int HERMES_DECL(MPI_File_iwrite_at)(MPI_File fh, MPI_Offset offset, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); fs_api->AWrite(f, stat_exists, buf, offset, count, datatype, request); return MPI_SUCCESS; } - return real_api->MPI_File_iwrite_at(fh, offset, buf, - count, datatype, request); + return real_api->MPI_File_iwrite_at(fh, offset, buf, count, datatype, + request); } int HERMES_DECL(MPI_File_iwrite)(MPI_File fh, const void *buf, int count, @@ -377,7 +418,9 @@ int HERMES_DECL(MPI_File_iwrite)(MPI_File fh, const void *buf, int count, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); fs_api->AWrite(f, stat_exists, buf, count, datatype, request); return MPI_SUCCESS; } @@ -390,12 +433,13 @@ int HERMES_DECL(MPI_File_iwrite_shared)(MPI_File fh, const void *buf, int count, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); fs_api->AWriteOrdered(f, stat_exists, buf, count, datatype, request); return MPI_SUCCESS; } - return real_api->MPI_File_iwrite_shared(fh, buf, count, - datatype, request); + return real_api->MPI_File_iwrite_shared(fh, buf, count, datatype, request); } /** @@ -406,9 +450,13 @@ int HERMES_DECL(MPI_File_sync)(MPI_File fh) { auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + File f; + f.mpi_fh_ = fh; + fs_api->_InitFile(f); fs_api->Sync(f, stat_exists); return 0; } return real_api->MPI_File_sync(fh); } + +} // extern C diff --git a/adapter/stdio/stdio.cc b/adapter/stdio/stdio.cc index e1235553b..4805cd5e4 100644 --- a/adapter/stdio/stdio.cc +++ b/adapter/stdio/stdio.cc @@ -35,6 +35,7 @@ using hermes::u64; namespace hapi = hermes::api; namespace stdfs = std::experimental::filesystem; +extern "C" { /** * MPI @@ -70,9 +71,13 @@ FILE *reopen_internal(const std::string &user_path, const char *mode, auto mdm = Singleton::GetInstance(); FILE *ret; ret = real_api->freopen(user_path.c_str(), mode, stream); - if (!ret) { return ret; } + if (!ret) { + return ret; + } - File f; f.fh_ = ret; fs_api->_InitFile(f); + File f; + f.fh_ = ret; + fs_api->_InitFile(f); std::string path_str = WeaklyCanonical(user_path).string(); LOG(INFO) << "Reopen file for filename " << path_str << " in mode " << mode << std::endl; @@ -109,7 +114,8 @@ FILE *HERMES_DECL(fopen64)(const char *path, const char *mode) { auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(path)) { LOG(INFO) << "Intercepting fopen64(" << path << ", " << mode << ")\n"; - AdapterStat stat; stat.mode_str = mode; + AdapterStat stat; + stat.mode_str = mode; return fs_api->Open(stat, path).fh_; } else { return real_api->fopen64(path, mode); @@ -123,8 +129,11 @@ FILE *HERMES_DECL(fdopen)(int fd, const char *mode) { if (ret && hermes::adapter::IsTracked(ret)) { LOG(INFO) << "Intercepting fdopen(" << fd << ", " << mode << ")\n"; std::string path_str = hermes::adapter::GetFilenameFromFD(fd); - File f; f.fh_ = ret; fs_api->_InitFile(f); - AdapterStat stat; stat.mode_str = mode; + File f; + f.fh_ = ret; + fs_api->_InitFile(f); + AdapterStat stat; + stat.mode_str = mode; fs_api->Open(stat, f, path_str); } return ret; @@ -155,7 +164,9 @@ int HERMES_DECL(fflush)(FILE *fp) { auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (fp && hermes::adapter::IsTracked(fp)) { - File f; f.fh_ = fp; fs_api->_InitFile(f); + File f; + f.fh_ = fp; + fs_api->_InitFile(f); return fs_api->Sync(f, stat_exists); } return real_api->fflush(fp); @@ -167,7 +178,9 @@ int HERMES_DECL(fclose)(FILE *fp) { auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp)) { LOG(INFO) << "Intercepting fclose(" << fp << ")\n"; - File f; f.fh_ = fp; fs_api->_InitFile(f); + File f; + f.fh_ = fp; + fs_api->_InitFile(f); int ret = fs_api->Close(f, stat_exists); if (stat_exists) return ret; } @@ -180,11 +193,16 @@ size_t HERMES_DECL(fwrite)(const void *ptr, size_t size, size_t nmemb, auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp)) { - LOG(INFO) << "Intercepting fwrite(" << ptr << ", " << size << ", " - << nmemb << ", " << fp << ")\n"; - File f; f.fh_ = fp; fs_api->_InitFile(f); IoStatus io_status; - size_t ret = fs_api->Write(f, stat_exists, ptr, size*nmemb, io_status); - if (stat_exists) { return ret; } + LOG(INFO) << "Intercepting fwrite(" << ptr << ", " << size << ", " << nmemb + << ", " << fp << ")\n"; + File f; + f.fh_ = fp; + fs_api->_InitFile(f); + IoStatus io_status; + size_t ret = fs_api->Write(f, stat_exists, ptr, size * nmemb, io_status); + if (stat_exists) { + return ret; + } } return real_api->fwrite(ptr, size, nmemb, fp); } @@ -194,10 +212,15 @@ int HERMES_DECL(fputc)(int c, FILE *fp) { auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp)) { - LOG(INFO) << "Intercepting fputc(" << c << ", " << fp << ")\n"; - File f; f.fh_ = fp; fs_api->_InitFile(f); IoStatus io_status; - fs_api->Write(f, stat_exists, &c, 1, io_status); - if (stat_exists) { return c; } + LOG(INFO) << "Intercepting fputc(" << c << ", " << fp << ")\n"; + File f; + f.fh_ = fp; + fs_api->_InitFile(f); + IoStatus io_status; + fs_api->Write(f, stat_exists, &c, 1, io_status); + if (stat_exists) { + return c; + } } return real_api->fputc(c, fp); } @@ -207,7 +230,9 @@ int HERMES_DECL(fgetpos)(FILE *fp, fpos_t *pos) { auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp) && pos) { - File f; f.fh_ = fp; fs_api->_InitFile(f); + File f; + f.fh_ = fp; + fs_api->_InitFile(f); LOG(INFO) << "Intercept fgetpos." << std::endl; // TODO(chogan): @portability In the GNU C Library, fpos_t is an opaque // data structure that contains internal data to represent file offset and @@ -215,7 +240,9 @@ int HERMES_DECL(fgetpos)(FILE *fp, fpos_t *pos) { // different internal representation. This will need to change to support // other compilers. pos->__pos = fs_api->Tell(f, stat_exists); - if (stat_exists) { return 0; } + if (stat_exists) { + return 0; + } } return real_api->fgetpos(fp, pos); } @@ -225,7 +252,9 @@ int HERMES_DECL(fgetpos64)(FILE *fp, fpos64_t *pos) { auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp) && pos) { - File f; f.fh_ = fp; fs_api->_InitFile(f); + File f; + f.fh_ = fp; + fs_api->_InitFile(f); LOG(INFO) << "Intercept fgetpos64." << std::endl; // TODO(chogan): @portability In the GNU C Library, fpos_t is an opaque // data structure that contains internal data to represent file offset and @@ -233,7 +262,9 @@ int HERMES_DECL(fgetpos64)(FILE *fp, fpos64_t *pos) { // different internal representation. This will need to change to support // other compilers. pos->__pos = fs_api->Tell(f, stat_exists); - if (stat_exists) { return 0; } + if (stat_exists) { + return 0; + } } return real_api->fgetpos64(fp, pos); } @@ -243,10 +274,15 @@ int HERMES_DECL(putc)(int c, FILE *fp) { auto real_api = Singleton::GetInstance(); auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp)) { - File f; f.fh_ = fp; fs_api->_InitFile(f); IoStatus io_status; + File f; + f.fh_ = fp; + fs_api->_InitFile(f); + IoStatus io_status; LOG(INFO) << "Intercept putc." << std::endl; fs_api->Write(f, stat_exists, &c, 1, io_status); - if (stat_exists) { return c; } + if (stat_exists) { + return c; + } } return real_api->fputc(c, fp); } @@ -257,7 +293,10 @@ int HERMES_DECL(putw)(int w, FILE *fp) { auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp)) { LOG(INFO) << "Intercept putw." << std::endl; - File f; f.fh_ = fp; fs_api->_InitFile(f); IoStatus io_status; + File f; + f.fh_ = fp; + fs_api->_InitFile(f); + IoStatus io_status; int ret = fs_api->Write(f, stat_exists, &w, sizeof(w), io_status); if (ret == sizeof(w)) { return 0; @@ -274,11 +313,16 @@ int HERMES_DECL(fputs)(const char *s, FILE *stream) { auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { LOG(INFO) << "Intercept fputs." << std::endl; - File f; f.fh_ = stream; fs_api->_InitFile(f); IoStatus io_status; + File f; + f.fh_ = stream; + fs_api->_InitFile(f); + IoStatus io_status; int ret = fs_api->Write(f, stat_exists, s, strlen(s), io_status); - if (stat_exists) { return ret; } + if (stat_exists) { + return ret; + } } - return real_api->fputs(s, stream);; + return real_api->fputs(s, stream); } size_t HERMES_DECL(fread)(void *ptr, size_t size, size_t nmemb, FILE *stream) { @@ -287,9 +331,14 @@ size_t HERMES_DECL(fread)(void *ptr, size_t size, size_t nmemb, FILE *stream) { auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { LOG(INFO) << "Intercept fread with size: " << size << "." << std::endl; - File f; f.fh_ = stream; fs_api->_InitFile(f); IoStatus io_status; + File f; + f.fh_ = stream; + fs_api->_InitFile(f); + IoStatus io_status; size_t ret = fs_api->Read(f, stat_exists, ptr, size * nmemb, io_status); - if (stat_exists) { return ret; } + if (stat_exists) { + return ret; + } } return real_api->fread(ptr, size, nmemb, stream); } @@ -300,10 +349,15 @@ int HERMES_DECL(fgetc)(FILE *stream) { auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { LOG(INFO) << "Intercept fgetc." << std::endl; - File f; f.fh_ = stream; fs_api->_InitFile(f); IoStatus io_status; + File f; + f.fh_ = stream; + fs_api->_InitFile(f); + IoStatus io_status; u8 value; fs_api->Read(f, stat_exists, &value, sizeof(u8), io_status); - if (stat_exists) { return value; } + if (stat_exists) { + return value; + } } return real_api->fgetc(stream); } @@ -314,10 +368,15 @@ int HERMES_DECL(getc)(FILE *stream) { auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { LOG(INFO) << "Intercept getc." << std::endl; - File f; f.fh_ = stream; fs_api->_InitFile(f); IoStatus io_status; + File f; + f.fh_ = stream; + fs_api->_InitFile(f); + IoStatus io_status; u8 value; fs_api->Read(f, stat_exists, &value, sizeof(u8), io_status); - if (stat_exists) { return value; } + if (stat_exists) { + return value; + } } return real_api->getc(stream); } @@ -328,10 +387,15 @@ int HERMES_DECL(getw)(FILE *stream) { auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { LOG(INFO) << "Intercept getw." << std::endl; - File f; f.fh_ = stream; fs_api->_InitFile(f); IoStatus io_status; + File f; + f.fh_ = stream; + fs_api->_InitFile(f); + IoStatus io_status; int value; fs_api->Read(f, stat_exists, &value, sizeof(int), io_status); - if (stat_exists) { return value; } + if (stat_exists) { + return value; + } } return real_api->getc(stream); } @@ -342,7 +406,10 @@ char *HERMES_DECL(fgets)(char *s, int size, FILE *stream) { auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { LOG(INFO) << "Intercept fgets." << std::endl; - File f; f.fh_ = stream; fs_api->_InitFile(f); IoStatus io_status; + File f; + f.fh_ = stream; + fs_api->_InitFile(f); + IoStatus io_status; size_t read_size = size - 1; size_t ret_size = fs_api->Read(f, stat_exists, s, read_size, io_status); if (ret_size < read_size) { @@ -374,9 +441,13 @@ void HERMES_DECL(rewind)(FILE *stream) { auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { LOG(INFO) << "Intercept rewind." << std::endl; - File f; f.fh_ = stream; fs_api->_InitFile(f); + File f; + f.fh_ = stream; + fs_api->_InitFile(f); fs_api->Seek(f, stat_exists, SeekMode::kSet, 0); - if (stat_exists) { return; } + if (stat_exists) { + return; + } } real_api->rewind(stream); } @@ -388,10 +459,14 @@ int HERMES_DECL(fseek)(FILE *stream, long offset, int whence) { if (hermes::adapter::IsTracked(stream)) { LOG(INFO) << "Intercept fseek offset:" << offset << " whence:" << whence << "." << std::endl; - File f; f.fh_ = stream; fs_api->_InitFile(f); - off_t ret = fs_api->Seek(f, stat_exists, - static_cast(whence), offset); - if (stat_exists && ret > 0) { return 0; } + File f; + f.fh_ = stream; + fs_api->_InitFile(f); + off_t ret = + fs_api->Seek(f, stat_exists, static_cast(whence), offset); + if (stat_exists && ret > 0) { + return 0; + } } return real_api->fseek(stream, offset, whence); } @@ -403,10 +478,14 @@ int HERMES_DECL(fseeko)(FILE *stream, off_t offset, int whence) { if (hermes::adapter::IsTracked(stream)) { LOG(INFO) << "Intercept fseeko offset:" << offset << " whence:" << whence << "." << std::endl; - File f; f.fh_ = stream; fs_api->_InitFile(f); - off_t ret = fs_api->Seek(f, stat_exists, - static_cast(whence), offset); - if (stat_exists && ret > 0) { return 0; } + File f; + f.fh_ = stream; + fs_api->_InitFile(f); + off_t ret = + fs_api->Seek(f, stat_exists, static_cast(whence), offset); + if (stat_exists && ret > 0) { + return 0; + } } return real_api->fseeko(stream, offset, whence); } @@ -418,10 +497,14 @@ int HERMES_DECL(fseeko64)(FILE *stream, off64_t offset, int whence) { if (hermes::adapter::IsTracked(stream)) { LOG(INFO) << "Intercept fseeko offset:" << offset << " whence:" << whence << "." << std::endl; - File f; f.fh_ = stream; fs_api->_InitFile(f); - off_t ret = fs_api->Seek(f, stat_exists, - static_cast(whence), offset); - if (stat_exists && ret > 0) { return 0; } + File f; + f.fh_ = stream; + fs_api->_InitFile(f); + off_t ret = + fs_api->Seek(f, stat_exists, static_cast(whence), offset); + if (stat_exists && ret > 0) { + return 0; + } } return real_api->fseeko64(stream, offset, whence); } @@ -433,9 +516,13 @@ int HERMES_DECL(fsetpos)(FILE *stream, const fpos_t *pos) { off_t offset = pos->__pos; if (hermes::adapter::IsTracked(stream)) { LOG(INFO) << "Intercept fsetpos offset:" << offset << "." << std::endl; - File f; f.fh_ = stream; fs_api->_InitFile(f); + File f; + f.fh_ = stream; + fs_api->_InitFile(f); off_t ret = fs_api->Seek(f, stat_exists, SeekMode::kSet, offset); - if (stat_exists && ret > 0) { return 0; } + if (stat_exists && ret > 0) { + return 0; + } } return real_api->fsetpos(stream, pos); } @@ -447,9 +534,13 @@ int HERMES_DECL(fsetpos64)(FILE *stream, const fpos64_t *pos) { off_t offset = pos->__pos; if (hermes::adapter::IsTracked(stream)) { LOG(INFO) << "Intercept fsetpos64 offset:" << offset << "." << std::endl; - File f; f.fh_ = stream; fs_api->_InitFile(f); + File f; + f.fh_ = stream; + fs_api->_InitFile(f); off_t ret = fs_api->Seek(f, stat_exists, SeekMode::kSet, offset); - if (stat_exists && ret > 0) { return 0; } + if (stat_exists && ret > 0) { + return 0; + } } return real_api->fsetpos64(stream, pos); } @@ -460,9 +551,15 @@ long int HERMES_DECL(ftell)(FILE *fp) { auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp)) { LOG(INFO) << "Intercept ftell." << std::endl; - File f; f.fh_ = fp; fs_api->_InitFile(f); + File f; + f.fh_ = fp; + fs_api->_InitFile(f); off_t ret = fs_api->Tell(f, stat_exists); - if (stat_exists) { return ret; } + if (stat_exists) { + return ret; + } } return real_api->ftell(fp); } + +} // extern C diff --git a/ci/lint.sh b/ci/lint.sh new file mode 100644 index 000000000..b35b21522 --- /dev/null +++ b/ci/lint.sh @@ -0,0 +1,10 @@ +HERMES_ROOT=$1 +ADAPTER=${HERMES_ROOT}/adapter + +cpplint --recursive \ +--exclude=${HERMES_ROOT}/src/stb_ds.h \ +--exclude=${ADAPTER}/posix/real_api.h \ +--exclude=${ADAPTER}/stdio/real_api.h \ +--exclude=${ADAPTER}/mpiio/real_api.h \ +${HERMES_ROOT}/adapter ${HERMES_ROOT}/benchmarks ${HERMES_ROOT}/data_stager \ +${HERMES_ROOT}/src ${HERMES_ROOT}/test From 8c86d97fa8d92b3db574b73efad177b54fccb292 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 18 Oct 2022 12:55:55 -0500 Subject: [PATCH 7/8] Ensure that all intercepted C functions and types are encapsulated in extern C --- adapter/adapter_generator/mpiio.py | 5 +- adapter/adapter_generator/stdio.py | 5 +- adapter/mpiio/real_api.h | 460 ++++++++--------------------- adapter/stdio/real_api.h | 206 +++---------- 4 files changed, 170 insertions(+), 506 deletions(-) diff --git a/adapter/adapter_generator/mpiio.py b/adapter/adapter_generator/mpiio.py index 8bbaa02b2..39df5e39d 100644 --- a/adapter/adapter_generator/mpiio.py +++ b/adapter/adapter_generator/mpiio.py @@ -33,7 +33,10 @@ includes = [ "", - "" + "", + "\"interceptor.h\"", + "\"filesystem/filesystem.h\"", + "\"filesystem/metadata_manager.h\"" ] ApiClass("mpiio", apis, includes) \ No newline at end of file diff --git a/adapter/adapter_generator/stdio.py b/adapter/adapter_generator/stdio.py index cc4413fa8..b8de6ba05 100644 --- a/adapter/adapter_generator/stdio.py +++ b/adapter/adapter_generator/stdio.py @@ -32,7 +32,10 @@ ] includes = [ - "\"cstdio.h\"" + "", + "\"interceptor.h\"", + "\"filesystem/filesystem.h\"", + "\"filesystem/metadata_manager.h\"" ] ApiClass("stdio", apis, includes) \ No newline at end of file diff --git a/adapter/mpiio/real_api.h b/adapter/mpiio/real_api.h index d95bfe2c8..58e21a15a 100644 --- a/adapter/mpiio/real_api.h +++ b/adapter/mpiio/real_api.h @@ -18,451 +18,221 @@ #include #include #include +#include "interceptor.h" +#include "filesystem/filesystem.h" +#include "filesystem/metadata_manager.h" + +#define REQUIRE_API(api_name) \ + if (real_api->api_name == nullptr) { \ + LOG(FATAL) << "HERMES Adapter failed to map symbol: " \ + #api_name << std::endl; \ + exit(1); + +extern "C" { +typedef int (*MPI_Init_t)(int * argc, char *** argv); +typedef int (*MPI_Finalize_t)( void); +typedef int (*MPI_Wait_t)(MPI_Request * req, MPI_Status * status); +typedef int (*MPI_Waitall_t)(int count, MPI_Request * req, MPI_Status * status); +typedef int (*MPI_File_open_t)(MPI_Comm comm, const char * filename, int amode, MPI_Info info, MPI_File * fh); +typedef int (*MPI_File_close_t)(MPI_File * fh); +typedef int (*MPI_File_seek_shared_t)(MPI_File fh, MPI_Offset offset, int whence); +typedef int (*MPI_File_seek_t)(MPI_File fh, MPI_Offset offset, int whence); +typedef int (*MPI_File_get_position_t)(MPI_File fh, MPI_Offset * offset); +typedef int (*MPI_File_read_all_t)(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Status * status); +typedef int (*MPI_File_read_at_all_t)(MPI_File fh, MPI_Offset offset, void * buf, int count, MPI_Datatype datatype, MPI_Status * status); +typedef int (*MPI_File_read_at_t)(MPI_File fh, MPI_Offset offset, void * buf, int count, MPI_Datatype datatype, MPI_Status * status); +typedef int (*MPI_File_read_t)(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Status * status); +typedef int (*MPI_File_read_ordered_t)(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Status * status); +typedef int (*MPI_File_read_shared_t)(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Status * status); +typedef int (*MPI_File_write_all_t)(MPI_File fh, const void * buf, int count, MPI_Datatype datatype, MPI_Status * status); +typedef int (*MPI_File_write_at_all_t)(MPI_File fh, MPI_Offset offset, const void * buf, int count, MPI_Datatype datatype, MPI_Status * status); +typedef int (*MPI_File_write_at_t)(MPI_File fh, MPI_Offset offset, const void * buf, int count, MPI_Datatype datatype, MPI_Status * status); +typedef int (*MPI_File_write_t)(MPI_File fh, const void * buf, int count, MPI_Datatype datatype, MPI_Status * status); +typedef int (*MPI_File_write_ordered_t)(MPI_File fh, const void * buf, int count, MPI_Datatype datatype, MPI_Status * status); +typedef int (*MPI_File_write_shared_t)(MPI_File fh, const void * buf, int count, MPI_Datatype datatype, MPI_Status * status); +typedef int (*MPI_File_iread_at_t)(MPI_File fh, MPI_Offset offset, void * buf, int count, MPI_Datatype datatype, MPI_Request * request); +typedef int (*MPI_File_iread_t)(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Request * request); +typedef int (*MPI_File_iread_shared_t)(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Request * request); +typedef int (*MPI_File_iwrite_at_t)(MPI_File fh, MPI_Offset offset, const void * buf, int count, MPI_Datatype datatype, MPI_Request * request); +typedef int (*MPI_File_iwrite_t)(MPI_File fh, const void * buf, int count, MPI_Datatype datatype, MPI_Request * request); +typedef int (*MPI_File_iwrite_shared_t)(MPI_File fh, const void * buf, int count, MPI_Datatype datatype, MPI_Request * request); +typedef int (*MPI_File_sync_t)(MPI_File fh); +} namespace hermes::adapter::mpiio { class API { public: - typedef int (*MPI_Init_t)(int * argc, char *** argv); int (*MPI_Init)(int * argc, char *** argv) = nullptr; - typedef int (*MPI_Finalize_t)( void); int (*MPI_Finalize)( void) = nullptr; - typedef int (*MPI_Wait_t)(MPI_Request * req, MPI_Status * status); int (*MPI_Wait)(MPI_Request * req, MPI_Status * status) = nullptr; - typedef int (*MPI_Waitall_t)(int count, MPI_Request * req, - MPI_Status * status); - int (*MPI_Waitall)(int count, MPI_Request * req, - MPI_Status * status) = nullptr; - typedef int (*MPI_File_open_t)(MPI_Comm comm, const char * filename, - int amode, MPI_Info info, MPI_File * fh); - int (*MPI_File_open)(MPI_Comm comm, const char * filename, int amode, - MPI_Info info, MPI_File * fh) = nullptr; - typedef int (*MPI_File_close_t)(MPI_File * fh); + int (*MPI_Waitall)(int count, MPI_Request * req, MPI_Status * status) = nullptr; + int (*MPI_File_open)(MPI_Comm comm, const char * filename, int amode, MPI_Info info, MPI_File * fh) = nullptr; int (*MPI_File_close)(MPI_File * fh) = nullptr; - typedef int (*MPI_File_seek_shared_t)(MPI_File fh, - MPI_Offset offset, int whence); - int (*MPI_File_seek_shared)(MPI_File fh, MPI_Offset offset, - int whence) = nullptr; - typedef int (*MPI_File_seek_t)(MPI_File fh, MPI_Offset offset, int whence); + int (*MPI_File_seek_shared)(MPI_File fh, MPI_Offset offset, int whence) = nullptr; int (*MPI_File_seek)(MPI_File fh, MPI_Offset offset, int whence) = nullptr; - typedef int (*MPI_File_get_position_t)(MPI_File fh, MPI_Offset * offset); int (*MPI_File_get_position)(MPI_File fh, MPI_Offset * offset) = nullptr; - typedef int (*MPI_File_read_all_t)(MPI_File fh, void * buf, int count, - MPI_Datatype datatype, - MPI_Status * status); - int (*MPI_File_read_all)(MPI_File fh, void * buf, int count, - MPI_Datatype datatype, - MPI_Status * status) = nullptr; - typedef int (*MPI_File_read_at_all_t)(MPI_File fh, MPI_Offset offset, - void * buf, int count, - MPI_Datatype datatype, - MPI_Status * status); - int (*MPI_File_read_at_all)(MPI_File fh, MPI_Offset offset, void * buf, - int count, MPI_Datatype datatype, - MPI_Status * status) = nullptr; - typedef int (*MPI_File_read_at_t)(MPI_File fh, MPI_Offset offset, - void * buf, int count, - MPI_Datatype datatype, MPI_Status * status); - int (*MPI_File_read_at)(MPI_File fh, MPI_Offset offset, - void * buf, int count, MPI_Datatype datatype, - MPI_Status * status) = nullptr; - typedef int (*MPI_File_read_t)(MPI_File fh, void * buf, int count, - MPI_Datatype datatype, MPI_Status * status); - int (*MPI_File_read)(MPI_File fh, void * buf, int count, - MPI_Datatype datatype, MPI_Status * status) = nullptr; - typedef int (*MPI_File_read_ordered_t)(MPI_File fh, void * buf, int count, - MPI_Datatype datatype, - MPI_Status * status); - int (*MPI_File_read_ordered)(MPI_File fh, void * buf, int count, - MPI_Datatype datatype, - MPI_Status * status) = nullptr; - typedef int (*MPI_File_read_shared_t)(MPI_File fh, void * buf, - int count, MPI_Datatype datatype, - MPI_Status * status); - int (*MPI_File_read_shared)(MPI_File fh, void * buf, int count, - MPI_Datatype datatype, - MPI_Status * status) = nullptr; - typedef int (*MPI_File_write_all_t)(MPI_File fh, const void * buf, - int count, MPI_Datatype datatype, - MPI_Status * status); - int (*MPI_File_write_all)(MPI_File fh, const void * buf, - int count, MPI_Datatype datatype, - MPI_Status * status) = nullptr; - typedef int (*MPI_File_write_at_all_t)(MPI_File fh, MPI_Offset offset, - const void * buf, int count, - MPI_Datatype datatype, - MPI_Status * status); - int (*MPI_File_write_at_all)(MPI_File fh, MPI_Offset offset, - const void * buf, int count, - MPI_Datatype datatype, - MPI_Status * status) = nullptr; - typedef int (*MPI_File_write_at_t)(MPI_File fh, MPI_Offset offset, - const void * buf, int count, - MPI_Datatype datatype, - MPI_Status * status); - int (*MPI_File_write_at)(MPI_File fh, MPI_Offset offset, const void * buf, - int count, MPI_Datatype datatype, - MPI_Status * status) = nullptr; - typedef int (*MPI_File_write_t)(MPI_File fh, const void * buf, int count, - MPI_Datatype datatype, MPI_Status * status); - int (*MPI_File_write)(MPI_File fh, const void * buf, int count, - MPI_Datatype datatype, MPI_Status * status) = nullptr; - typedef int (*MPI_File_write_ordered_t)(MPI_File fh, const void * buf, - int count, MPI_Datatype datatype, - MPI_Status * status); - int (*MPI_File_write_ordered)(MPI_File fh, const void * buf, int count, - MPI_Datatype datatype, - MPI_Status * status) = nullptr; - typedef int (*MPI_File_write_shared_t)(MPI_File fh, const void * buf, - int count, MPI_Datatype datatype, - MPI_Status * status); - int (*MPI_File_write_shared)(MPI_File fh, const void * buf, int count, - MPI_Datatype datatype, - MPI_Status * status) = nullptr; - typedef int (*MPI_File_iread_at_t)(MPI_File fh, MPI_Offset offset, - void * buf, int count, - MPI_Datatype datatype, - MPI_Request * request); - int (*MPI_File_iread_at)(MPI_File fh, MPI_Offset offset, void * buf, - int count, MPI_Datatype datatype, - MPI_Request * request) = nullptr; - typedef int (*MPI_File_iread_t)(MPI_File fh, void * buf, int count, - MPI_Datatype datatype, MPI_Request * request); - int (*MPI_File_iread)(MPI_File fh, void * buf, int count, - MPI_Datatype datatype, MPI_Request * request) = nullptr; - typedef int (*MPI_File_iread_shared_t)(MPI_File fh, void * buf, int count, - MPI_Datatype datatype, - MPI_Request * request); - int (*MPI_File_iread_shared)(MPI_File fh, void * buf, int count, - MPI_Datatype datatype, - MPI_Request * request) = nullptr; - typedef int (*MPI_File_iwrite_at_t)(MPI_File fh, MPI_Offset offset, - const void * buf, int count, - MPI_Datatype datatype, - MPI_Request * request); - int (*MPI_File_iwrite_at)(MPI_File fh, MPI_Offset offset, const void * buf, - int count, MPI_Datatype datatype, - MPI_Request * request) = nullptr; - typedef int (*MPI_File_iwrite_t)(MPI_File fh, const void * buf, int count, - MPI_Datatype datatype, - MPI_Request * request); - int (*MPI_File_iwrite)(MPI_File fh, const void * buf, int count, - MPI_Datatype datatype, - MPI_Request * request) = nullptr; - typedef int (*MPI_File_iwrite_shared_t)(MPI_File fh, const void * buf, - int count, MPI_Datatype datatype, - MPI_Request * request); - int (*MPI_File_iwrite_shared)(MPI_File fh, const void * buf, int count, - MPI_Datatype datatype, - MPI_Request * request) = nullptr; - typedef int (*MPI_File_sync_t)(MPI_File fh); + int (*MPI_File_read_all)(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Status * status) = nullptr; + int (*MPI_File_read_at_all)(MPI_File fh, MPI_Offset offset, void * buf, int count, MPI_Datatype datatype, MPI_Status * status) = nullptr; + int (*MPI_File_read_at)(MPI_File fh, MPI_Offset offset, void * buf, int count, MPI_Datatype datatype, MPI_Status * status) = nullptr; + int (*MPI_File_read)(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Status * status) = nullptr; + int (*MPI_File_read_ordered)(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Status * status) = nullptr; + int (*MPI_File_read_shared)(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Status * status) = nullptr; + int (*MPI_File_write_all)(MPI_File fh, const void * buf, int count, MPI_Datatype datatype, MPI_Status * status) = nullptr; + int (*MPI_File_write_at_all)(MPI_File fh, MPI_Offset offset, const void * buf, int count, MPI_Datatype datatype, MPI_Status * status) = nullptr; + int (*MPI_File_write_at)(MPI_File fh, MPI_Offset offset, const void * buf, int count, MPI_Datatype datatype, MPI_Status * status) = nullptr; + int (*MPI_File_write)(MPI_File fh, const void * buf, int count, MPI_Datatype datatype, MPI_Status * status) = nullptr; + int (*MPI_File_write_ordered)(MPI_File fh, const void * buf, int count, MPI_Datatype datatype, MPI_Status * status) = nullptr; + int (*MPI_File_write_shared)(MPI_File fh, const void * buf, int count, MPI_Datatype datatype, MPI_Status * status) = nullptr; + int (*MPI_File_iread_at)(MPI_File fh, MPI_Offset offset, void * buf, int count, MPI_Datatype datatype, MPI_Request * request) = nullptr; + int (*MPI_File_iread)(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Request * request) = nullptr; + int (*MPI_File_iread_shared)(MPI_File fh, void * buf, int count, MPI_Datatype datatype, MPI_Request * request) = nullptr; + int (*MPI_File_iwrite_at)(MPI_File fh, MPI_Offset offset, const void * buf, int count, MPI_Datatype datatype, MPI_Request * request) = nullptr; + int (*MPI_File_iwrite)(MPI_File fh, const void * buf, int count, MPI_Datatype datatype, MPI_Request * request) = nullptr; + int (*MPI_File_iwrite_shared)(MPI_File fh, const void * buf, int count, MPI_Datatype datatype, MPI_Request * request) = nullptr; int (*MPI_File_sync)(MPI_File fh) = nullptr; + API() { - void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, - "mpiio_intercepted"); + void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, "mpiio_intercepted"); if (is_intercepted) { MPI_Init = (MPI_Init_t)dlsym(RTLD_NEXT, "MPI_Init"); } else { MPI_Init = (MPI_Init_t)dlsym(RTLD_DEFAULT, "MPI_Init"); } - if (MPI_Init == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_Init" << std::endl; - } if (is_intercepted) { - MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_NEXT, - "MPI_Finalize"); + MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_NEXT, "MPI_Finalize"); } else { - MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_DEFAULT, - "MPI_Finalize"); - } - if (MPI_Finalize == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_Finalize" << std::endl; + MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_DEFAULT, "MPI_Finalize"); } if (is_intercepted) { MPI_Wait = (MPI_Wait_t)dlsym(RTLD_NEXT, "MPI_Wait"); } else { MPI_Wait = (MPI_Wait_t)dlsym(RTLD_DEFAULT, "MPI_Wait"); } - if (MPI_Wait == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_Wait" << std::endl; - } if (is_intercepted) { - MPI_Waitall = (MPI_Waitall_t)dlsym(RTLD_NEXT, - "MPI_Waitall"); + MPI_Waitall = (MPI_Waitall_t)dlsym(RTLD_NEXT, "MPI_Waitall"); } else { - MPI_Waitall = (MPI_Waitall_t)dlsym(RTLD_DEFAULT, - "MPI_Waitall"); - } - if (MPI_Waitall == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_Waitall" << std::endl; + MPI_Waitall = (MPI_Waitall_t)dlsym(RTLD_DEFAULT, "MPI_Waitall"); } if (is_intercepted) { - MPI_File_open = (MPI_File_open_t)dlsym(RTLD_NEXT, - "MPI_File_open"); + MPI_File_open = (MPI_File_open_t)dlsym(RTLD_NEXT, "MPI_File_open"); } else { - MPI_File_open = (MPI_File_open_t)dlsym(RTLD_DEFAULT, - "MPI_File_open"); - } - if (MPI_File_open == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_open" << std::endl; + MPI_File_open = (MPI_File_open_t)dlsym(RTLD_DEFAULT, "MPI_File_open"); } if (is_intercepted) { - MPI_File_close = (MPI_File_close_t)dlsym(RTLD_NEXT, - "MPI_File_close"); + MPI_File_close = (MPI_File_close_t)dlsym(RTLD_NEXT, "MPI_File_close"); } else { - MPI_File_close = (MPI_File_close_t)dlsym(RTLD_DEFAULT, - "MPI_File_close"); - } - if (MPI_File_close == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_close" << std::endl; + MPI_File_close = (MPI_File_close_t)dlsym(RTLD_DEFAULT, "MPI_File_close"); } if (is_intercepted) { - MPI_File_seek_shared = (MPI_File_seek_shared_t) - dlsym(RTLD_NEXT, "MPI_File_seek_shared"); + MPI_File_seek_shared = (MPI_File_seek_shared_t)dlsym(RTLD_NEXT, "MPI_File_seek_shared"); } else { - MPI_File_seek_shared = (MPI_File_seek_shared_t) - dlsym(RTLD_DEFAULT, "MPI_File_seek_shared"); - } - if (MPI_File_seek_shared == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_seek_shared" << std::endl; + MPI_File_seek_shared = (MPI_File_seek_shared_t)dlsym(RTLD_DEFAULT, "MPI_File_seek_shared"); } if (is_intercepted) { - MPI_File_seek = (MPI_File_seek_t) - dlsym(RTLD_NEXT, "MPI_File_seek"); + MPI_File_seek = (MPI_File_seek_t)dlsym(RTLD_NEXT, "MPI_File_seek"); } else { - MPI_File_seek = (MPI_File_seek_t) - dlsym(RTLD_DEFAULT, "MPI_File_seek"); - } - if (MPI_File_seek == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_seek" << std::endl; + MPI_File_seek = (MPI_File_seek_t)dlsym(RTLD_DEFAULT, "MPI_File_seek"); } if (is_intercepted) { - MPI_File_get_position = (MPI_File_get_position_t) - dlsym(RTLD_NEXT, "MPI_File_get_position"); + MPI_File_get_position = (MPI_File_get_position_t)dlsym(RTLD_NEXT, "MPI_File_get_position"); } else { - MPI_File_get_position = (MPI_File_get_position_t) - dlsym(RTLD_DEFAULT, "MPI_File_get_position"); - } - if (MPI_File_get_position == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_get_position" << std::endl; + MPI_File_get_position = (MPI_File_get_position_t)dlsym(RTLD_DEFAULT, "MPI_File_get_position"); } if (is_intercepted) { - MPI_File_read_all = (MPI_File_read_all_t) - dlsym(RTLD_NEXT, "MPI_File_read_all"); + MPI_File_read_all = (MPI_File_read_all_t)dlsym(RTLD_NEXT, "MPI_File_read_all"); } else { - MPI_File_read_all = (MPI_File_read_all_t) - dlsym(RTLD_DEFAULT, "MPI_File_read_all"); - } - if (MPI_File_read_all == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_read_all" << std::endl; + MPI_File_read_all = (MPI_File_read_all_t)dlsym(RTLD_DEFAULT, "MPI_File_read_all"); } if (is_intercepted) { - MPI_File_read_at_all = (MPI_File_read_at_all_t) - dlsym(RTLD_NEXT, "MPI_File_read_at_all"); + MPI_File_read_at_all = (MPI_File_read_at_all_t)dlsym(RTLD_NEXT, "MPI_File_read_at_all"); } else { - MPI_File_read_at_all = (MPI_File_read_at_all_t) - dlsym(RTLD_DEFAULT, "MPI_File_read_at_all"); - } - if (MPI_File_read_at_all == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_read_at_all" << std::endl; + MPI_File_read_at_all = (MPI_File_read_at_all_t)dlsym(RTLD_DEFAULT, "MPI_File_read_at_all"); } if (is_intercepted) { - MPI_File_read_at = (MPI_File_read_at_t) - dlsym(RTLD_NEXT, "MPI_File_read_at"); + MPI_File_read_at = (MPI_File_read_at_t)dlsym(RTLD_NEXT, "MPI_File_read_at"); } else { - MPI_File_read_at = (MPI_File_read_at_t) - dlsym(RTLD_DEFAULT, "MPI_File_read_at"); - } - if (MPI_File_read_at == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_read_at" << std::endl; + MPI_File_read_at = (MPI_File_read_at_t)dlsym(RTLD_DEFAULT, "MPI_File_read_at"); } if (is_intercepted) { - MPI_File_read = (MPI_File_read_t) - dlsym(RTLD_NEXT, "MPI_File_read"); + MPI_File_read = (MPI_File_read_t)dlsym(RTLD_NEXT, "MPI_File_read"); } else { - MPI_File_read = (MPI_File_read_t) - dlsym(RTLD_DEFAULT, "MPI_File_read"); - } - if (MPI_File_read == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_read" << std::endl; + MPI_File_read = (MPI_File_read_t)dlsym(RTLD_DEFAULT, "MPI_File_read"); } if (is_intercepted) { - MPI_File_read_ordered = (MPI_File_read_ordered_t) - dlsym(RTLD_NEXT, "MPI_File_read_ordered"); + MPI_File_read_ordered = (MPI_File_read_ordered_t)dlsym(RTLD_NEXT, "MPI_File_read_ordered"); } else { - MPI_File_read_ordered = (MPI_File_read_ordered_t) - dlsym(RTLD_DEFAULT, "MPI_File_read_ordered"); - } - if (MPI_File_read_ordered == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_read_ordered" << std::endl; + MPI_File_read_ordered = (MPI_File_read_ordered_t)dlsym(RTLD_DEFAULT, "MPI_File_read_ordered"); } if (is_intercepted) { - MPI_File_read_shared = (MPI_File_read_shared_t) - dlsym(RTLD_NEXT, "MPI_File_read_shared"); + MPI_File_read_shared = (MPI_File_read_shared_t)dlsym(RTLD_NEXT, "MPI_File_read_shared"); } else { - MPI_File_read_shared = (MPI_File_read_shared_t) - dlsym(RTLD_DEFAULT, "MPI_File_read_shared"); - } - if (MPI_File_read_shared == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_read_shared" << std::endl; + MPI_File_read_shared = (MPI_File_read_shared_t)dlsym(RTLD_DEFAULT, "MPI_File_read_shared"); } if (is_intercepted) { - MPI_File_write_all = (MPI_File_write_all_t) - dlsym(RTLD_NEXT, "MPI_File_write_all"); + MPI_File_write_all = (MPI_File_write_all_t)dlsym(RTLD_NEXT, "MPI_File_write_all"); } else { - MPI_File_write_all = (MPI_File_write_all_t) - dlsym(RTLD_DEFAULT, "MPI_File_write_all"); - } - if (MPI_File_write_all == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_write_all" << std::endl; + MPI_File_write_all = (MPI_File_write_all_t)dlsym(RTLD_DEFAULT, "MPI_File_write_all"); } if (is_intercepted) { - MPI_File_write_at_all = (MPI_File_write_at_all_t) - dlsym(RTLD_NEXT, "MPI_File_write_at_all"); + MPI_File_write_at_all = (MPI_File_write_at_all_t)dlsym(RTLD_NEXT, "MPI_File_write_at_all"); } else { - MPI_File_write_at_all = (MPI_File_write_at_all_t) - dlsym(RTLD_DEFAULT, "MPI_File_write_at_all"); - } - if (MPI_File_write_at_all == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_write_at_all" << std::endl; + MPI_File_write_at_all = (MPI_File_write_at_all_t)dlsym(RTLD_DEFAULT, "MPI_File_write_at_all"); } if (is_intercepted) { - MPI_File_write_at = (MPI_File_write_at_t) - dlsym(RTLD_NEXT, "MPI_File_write_at"); + MPI_File_write_at = (MPI_File_write_at_t)dlsym(RTLD_NEXT, "MPI_File_write_at"); } else { - MPI_File_write_at = (MPI_File_write_at_t) - dlsym(RTLD_DEFAULT, "MPI_File_write_at"); - } - if (MPI_File_write_at == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_write_at" << std::endl; + MPI_File_write_at = (MPI_File_write_at_t)dlsym(RTLD_DEFAULT, "MPI_File_write_at"); } if (is_intercepted) { - MPI_File_write = (MPI_File_write_t) - dlsym(RTLD_NEXT, "MPI_File_write"); + MPI_File_write = (MPI_File_write_t)dlsym(RTLD_NEXT, "MPI_File_write"); } else { - MPI_File_write = (MPI_File_write_t) - dlsym(RTLD_DEFAULT, "MPI_File_write"); - } - if (MPI_File_write == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_write" << std::endl; + MPI_File_write = (MPI_File_write_t)dlsym(RTLD_DEFAULT, "MPI_File_write"); } if (is_intercepted) { - MPI_File_write_ordered = (MPI_File_write_ordered_t) - dlsym(RTLD_NEXT, "MPI_File_write_ordered"); + MPI_File_write_ordered = (MPI_File_write_ordered_t)dlsym(RTLD_NEXT, "MPI_File_write_ordered"); } else { - MPI_File_write_ordered = (MPI_File_write_ordered_t) - dlsym(RTLD_DEFAULT, "MPI_File_write_ordered"); - } - if (MPI_File_write_ordered == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_write_ordered" << std::endl; + MPI_File_write_ordered = (MPI_File_write_ordered_t)dlsym(RTLD_DEFAULT, "MPI_File_write_ordered"); } if (is_intercepted) { - MPI_File_write_shared = (MPI_File_write_shared_t) - dlsym(RTLD_NEXT, "MPI_File_write_shared"); + MPI_File_write_shared = (MPI_File_write_shared_t)dlsym(RTLD_NEXT, "MPI_File_write_shared"); } else { - MPI_File_write_shared = (MPI_File_write_shared_t) - dlsym(RTLD_DEFAULT, "MPI_File_write_shared"); - } - if (MPI_File_write_shared == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_write_shared" << std::endl; + MPI_File_write_shared = (MPI_File_write_shared_t)dlsym(RTLD_DEFAULT, "MPI_File_write_shared"); } if (is_intercepted) { - MPI_File_iread_at = (MPI_File_iread_at_t) - dlsym(RTLD_NEXT, "MPI_File_iread_at"); + MPI_File_iread_at = (MPI_File_iread_at_t)dlsym(RTLD_NEXT, "MPI_File_iread_at"); } else { - MPI_File_iread_at = (MPI_File_iread_at_t) - dlsym(RTLD_DEFAULT, "MPI_File_iread_at"); - } - if (MPI_File_iread_at == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_iread_at" << std::endl; + MPI_File_iread_at = (MPI_File_iread_at_t)dlsym(RTLD_DEFAULT, "MPI_File_iread_at"); } if (is_intercepted) { - MPI_File_iread = (MPI_File_iread_t) - dlsym(RTLD_NEXT, "MPI_File_iread"); + MPI_File_iread = (MPI_File_iread_t)dlsym(RTLD_NEXT, "MPI_File_iread"); } else { - MPI_File_iread = (MPI_File_iread_t) - dlsym(RTLD_DEFAULT, "MPI_File_iread"); - } - if (MPI_File_iread == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_iread" << std::endl; + MPI_File_iread = (MPI_File_iread_t)dlsym(RTLD_DEFAULT, "MPI_File_iread"); } if (is_intercepted) { - MPI_File_iread_shared = (MPI_File_iread_shared_t) - dlsym(RTLD_NEXT, "MPI_File_iread_shared"); + MPI_File_iread_shared = (MPI_File_iread_shared_t)dlsym(RTLD_NEXT, "MPI_File_iread_shared"); } else { - MPI_File_iread_shared = (MPI_File_iread_shared_t) - dlsym(RTLD_DEFAULT, "MPI_File_iread_shared"); - } - if (MPI_File_iread_shared == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_iread_shared" << std::endl; + MPI_File_iread_shared = (MPI_File_iread_shared_t)dlsym(RTLD_DEFAULT, "MPI_File_iread_shared"); } if (is_intercepted) { - MPI_File_iwrite_at = (MPI_File_iwrite_at_t) - dlsym(RTLD_NEXT, "MPI_File_iwrite_at"); + MPI_File_iwrite_at = (MPI_File_iwrite_at_t)dlsym(RTLD_NEXT, "MPI_File_iwrite_at"); } else { - MPI_File_iwrite_at = (MPI_File_iwrite_at_t) - dlsym(RTLD_DEFAULT, "MPI_File_iwrite_at"); - } - if (MPI_File_iwrite_at == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_iwrite_at" << std::endl; + MPI_File_iwrite_at = (MPI_File_iwrite_at_t)dlsym(RTLD_DEFAULT, "MPI_File_iwrite_at"); } if (is_intercepted) { - MPI_File_iwrite = (MPI_File_iwrite_t) - dlsym(RTLD_NEXT, "MPI_File_iwrite"); + MPI_File_iwrite = (MPI_File_iwrite_t)dlsym(RTLD_NEXT, "MPI_File_iwrite"); } else { - MPI_File_iwrite = (MPI_File_iwrite_t) - dlsym(RTLD_DEFAULT, "MPI_File_iwrite"); - } - if (MPI_File_iwrite == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_iwrite" << std::endl; + MPI_File_iwrite = (MPI_File_iwrite_t)dlsym(RTLD_DEFAULT, "MPI_File_iwrite"); } if (is_intercepted) { - MPI_File_iwrite_shared = (MPI_File_iwrite_shared_t) - dlsym(RTLD_NEXT, "MPI_File_iwrite_shared"); + MPI_File_iwrite_shared = (MPI_File_iwrite_shared_t)dlsym(RTLD_NEXT, "MPI_File_iwrite_shared"); } else { - MPI_File_iwrite_shared = (MPI_File_iwrite_shared_t) - dlsym(RTLD_DEFAULT, "MPI_File_iwrite_shared"); - } - if (MPI_File_iwrite_shared == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_iwrite_shared" << std::endl; + MPI_File_iwrite_shared = (MPI_File_iwrite_shared_t)dlsym(RTLD_DEFAULT, "MPI_File_iwrite_shared"); } if (is_intercepted) { - MPI_File_sync = (MPI_File_sync_t) - dlsym(RTLD_NEXT, "MPI_File_sync"); + MPI_File_sync = (MPI_File_sync_t)dlsym(RTLD_NEXT, "MPI_File_sync"); } else { - MPI_File_sync = (MPI_File_sync_t) - dlsym(RTLD_DEFAULT, "MPI_File_sync"); - } - if (MPI_File_sync == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_File_sync" << std::endl; + MPI_File_sync = (MPI_File_sync_t)dlsym(RTLD_DEFAULT, "MPI_File_sync"); } } }; diff --git a/adapter/stdio/real_api.h b/adapter/stdio/real_api.h index c4d702452..4ff3ae343 100644 --- a/adapter/stdio/real_api.h +++ b/adapter/stdio/real_api.h @@ -16,335 +16,223 @@ #include #include #include +#include #include "interceptor.h" #include "filesystem/filesystem.h" -#include +#include "filesystem/metadata_manager.h" + +#define REQUIRE_API(api_name) \ + if (real_api->api_name == nullptr) { \ + LOG(FATAL) << "HERMES Adapter failed to map symbol: " \ + #api_name << std::endl; \ + exit(1); + +extern "C" { +typedef int (*MPI_Init_t)(int * argc, char *** argv); +typedef int (*MPI_Finalize_t)( void); +typedef FILE * (*fopen_t)(const char * path, const char * mode); +typedef FILE * (*fopen64_t)(const char * path, const char * mode); +typedef FILE * (*fdopen_t)(int fd, const char * mode); +typedef FILE * (*freopen_t)(const char * path, const char * mode, FILE * stream); +typedef FILE * (*freopen64_t)(const char * path, const char * mode, FILE * stream); +typedef int (*fflush_t)(FILE * fp); +typedef int (*fclose_t)(FILE * fp); +typedef size_t (*fwrite_t)(const void * ptr, size_t size, size_t nmemb, FILE * fp); +typedef int (*fputc_t)(int c, FILE * fp); +typedef int (*fgetpos_t)(FILE * fp, fpos_t * pos); +typedef int (*fgetpos64_t)(FILE * fp, fpos64_t * pos); +typedef int (*putc_t)(int c, FILE * fp); +typedef int (*putw_t)(int w, FILE * fp); +typedef int (*fputs_t)(const char * s, FILE * stream); +typedef size_t (*fread_t)(void * ptr, size_t size, size_t nmemb, FILE * stream); +typedef int (*fgetc_t)(FILE * stream); +typedef int (*getc_t)(FILE * stream); +typedef int (*getw_t)(FILE * stream); +typedef char * (*fgets_t)(char * s, int size, FILE * stream); +typedef void (*rewind_t)(FILE * stream); +typedef int (*fseek_t)(FILE * stream, long offset, int whence); +typedef int (*fseeko_t)(FILE * stream, off_t offset, int whence); +typedef int (*fseeko64_t)(FILE * stream, off64_t offset, int whence); +typedef int (*fsetpos_t)(FILE * stream, const fpos_t * pos); +typedef int (*fsetpos64_t)(FILE * stream, const fpos64_t * pos); +typedef long int (*ftell_t)(FILE * fp); +} namespace hermes::adapter::stdio { class API { public: - typedef int (*MPI_Init_t)(int * argc, char *** argv); int (*MPI_Init)(int * argc, char *** argv) = nullptr; - typedef int (*MPI_Finalize_t)( void); int (*MPI_Finalize)( void) = nullptr; - typedef FILE * (*fopen_t)(const char * path, const char * mode); FILE * (*fopen)(const char * path, const char * mode) = nullptr; - typedef FILE * (*fopen64_t)(const char * path, const char * mode); FILE * (*fopen64)(const char * path, const char * mode) = nullptr; - typedef FILE * (*fdopen_t)(int fd, const char * mode); FILE * (*fdopen)(int fd, const char * mode) = nullptr; - typedef FILE * (*freopen_t)(const char * path, - const char * mode, FILE * stream); - FILE * (*freopen)(const char * path, - const char * mode, FILE * stream) = nullptr; - typedef FILE * (*freopen64_t)(const char * path, - const char * mode, FILE * stream); - FILE * (*freopen64)(const char * path, - const char * mode, FILE * stream) = nullptr; - typedef int (*fflush_t)(FILE * fp); + FILE * (*freopen)(const char * path, const char * mode, FILE * stream) = nullptr; + FILE * (*freopen64)(const char * path, const char * mode, FILE * stream) = nullptr; int (*fflush)(FILE * fp) = nullptr; - typedef int (*fclose_t)(FILE * fp); int (*fclose)(FILE * fp) = nullptr; - typedef size_t (*fwrite_t)(const void * ptr, size_t size, - size_t nmemb, FILE * fp); - size_t (*fwrite)(const void * ptr, size_t size, - size_t nmemb, FILE * fp) = nullptr; - typedef int (*fputc_t)(int c, FILE * fp); + size_t (*fwrite)(const void * ptr, size_t size, size_t nmemb, FILE * fp) = nullptr; int (*fputc)(int c, FILE * fp) = nullptr; - typedef int (*fgetpos_t)(FILE * fp, fpos_t * pos); int (*fgetpos)(FILE * fp, fpos_t * pos) = nullptr; - typedef int (*fgetpos64_t)(FILE * fp, fpos64_t * pos); int (*fgetpos64)(FILE * fp, fpos64_t * pos) = nullptr; - typedef int (*putc_t)(int c, FILE * fp); int (*putc)(int c, FILE * fp) = nullptr; - typedef int (*putw_t)(int w, FILE * fp); int (*putw)(int w, FILE * fp) = nullptr; - typedef int (*fputs_t)(const char * s, FILE * stream); int (*fputs)(const char * s, FILE * stream) = nullptr; - typedef size_t (*fread_t)(void * ptr, size_t size, - size_t nmemb, FILE * stream); - size_t (*fread)(void * ptr, size_t size, - size_t nmemb, FILE * stream) = nullptr; - typedef int (*fgetc_t)(FILE * stream); + size_t (*fread)(void * ptr, size_t size, size_t nmemb, FILE * stream) = nullptr; int (*fgetc)(FILE * stream) = nullptr; - typedef int (*getc_t)(FILE * stream); int (*getc)(FILE * stream) = nullptr; - typedef int (*getw_t)(FILE * stream); int (*getw)(FILE * stream) = nullptr; - typedef char * (*fgets_t)(char * s, int size, FILE * stream); char * (*fgets)(char * s, int size, FILE * stream) = nullptr; - typedef void (*rewind_t)(FILE * stream); void (*rewind)(FILE * stream) = nullptr; - typedef int (*fseek_t)(FILE * stream, long offset, int whence); int (*fseek)(FILE * stream, long offset, int whence) = nullptr; - typedef int (*fseeko_t)(FILE * stream, off_t offset, int whence); int (*fseeko)(FILE * stream, off_t offset, int whence) = nullptr; - typedef int (*fseeko64_t)(FILE * stream, off64_t offset, int whence); int (*fseeko64)(FILE * stream, off64_t offset, int whence) = nullptr; - typedef int (*fsetpos_t)(FILE * stream, const fpos_t * pos); int (*fsetpos)(FILE * stream, const fpos_t * pos) = nullptr; - typedef int (*fsetpos64_t)(FILE * stream, const fpos64_t * pos); int (*fsetpos64)(FILE * stream, const fpos64_t * pos) = nullptr; - typedef long int (*ftell_t)(FILE * fp); long int (*ftell)(FILE * fp) = nullptr; + API() { - void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, - "stdio_intercepted"); + void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, "stdio_intercepted"); if (is_intercepted) { MPI_Init = (MPI_Init_t)dlsym(RTLD_NEXT, "MPI_Init"); } else { MPI_Init = (MPI_Init_t)dlsym(RTLD_DEFAULT, "MPI_Init"); } - if (MPI_Init == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_Init" << std::endl; - } if (is_intercepted) { - MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_NEXT, - "MPI_Finalize"); + MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_NEXT, "MPI_Finalize"); } else { - MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_DEFAULT, - "MPI_Finalize"); - } - if (MPI_Finalize == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "MPI_Finalize" << std::endl; + MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_DEFAULT, "MPI_Finalize"); } if (is_intercepted) { fopen = (fopen_t)dlsym(RTLD_NEXT, "fopen"); } else { fopen = (fopen_t)dlsym(RTLD_DEFAULT, "fopen"); } - if (fopen == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fopen" << std::endl; - } if (is_intercepted) { fopen64 = (fopen64_t)dlsym(RTLD_NEXT, "fopen64"); } else { fopen64 = (fopen64_t)dlsym(RTLD_DEFAULT, "fopen64"); } - if (fopen64 == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fopen64" << std::endl; - } if (is_intercepted) { fdopen = (fdopen_t)dlsym(RTLD_NEXT, "fdopen"); } else { fdopen = (fdopen_t)dlsym(RTLD_DEFAULT, "fdopen"); } - if (fdopen == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fdopen" << std::endl; - } if (is_intercepted) { freopen = (freopen_t)dlsym(RTLD_NEXT, "freopen"); } else { freopen = (freopen_t)dlsym(RTLD_DEFAULT, "freopen"); } - if (freopen == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "freopen" << std::endl; - } if (is_intercepted) { freopen64 = (freopen64_t)dlsym(RTLD_NEXT, "freopen64"); } else { freopen64 = (freopen64_t)dlsym(RTLD_DEFAULT, "freopen64"); } - if (freopen64 == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "freopen64" << std::endl; - } if (is_intercepted) { fflush = (fflush_t)dlsym(RTLD_NEXT, "fflush"); } else { fflush = (fflush_t)dlsym(RTLD_DEFAULT, "fflush"); } - if (fflush == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fflush" << std::endl; - } if (is_intercepted) { fclose = (fclose_t)dlsym(RTLD_NEXT, "fclose"); } else { fclose = (fclose_t)dlsym(RTLD_DEFAULT, "fclose"); } - if (fclose == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fclose" << std::endl; - } if (is_intercepted) { fwrite = (fwrite_t)dlsym(RTLD_NEXT, "fwrite"); } else { fwrite = (fwrite_t)dlsym(RTLD_DEFAULT, "fwrite"); } - if (fwrite == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fwrite" << std::endl; - } if (is_intercepted) { fputc = (fputc_t)dlsym(RTLD_NEXT, "fputc"); } else { fputc = (fputc_t)dlsym(RTLD_DEFAULT, "fputc"); } - if (fputc == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fputc" << std::endl; - } if (is_intercepted) { fgetpos = (fgetpos_t)dlsym(RTLD_NEXT, "fgetpos"); } else { fgetpos = (fgetpos_t)dlsym(RTLD_DEFAULT, "fgetpos"); } - if (fgetpos == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fgetpos" << std::endl; - } if (is_intercepted) { fgetpos64 = (fgetpos64_t)dlsym(RTLD_NEXT, "fgetpos64"); } else { fgetpos64 = (fgetpos64_t)dlsym(RTLD_DEFAULT, "fgetpos64"); } - if (fgetpos64 == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fgetpos64" << std::endl; - } if (is_intercepted) { putc = (putc_t)dlsym(RTLD_NEXT, "putc"); } else { putc = (putc_t)dlsym(RTLD_DEFAULT, "putc"); } - if (putc == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "putc" << std::endl; - } if (is_intercepted) { putw = (putw_t)dlsym(RTLD_NEXT, "putw"); } else { putw = (putw_t)dlsym(RTLD_DEFAULT, "putw"); } - if (putw == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "putw" << std::endl; - } if (is_intercepted) { fputs = (fputs_t)dlsym(RTLD_NEXT, "fputs"); } else { fputs = (fputs_t)dlsym(RTLD_DEFAULT, "fputs"); } - if (fputs == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fputs" << std::endl; - } if (is_intercepted) { fread = (fread_t)dlsym(RTLD_NEXT, "fread"); } else { fread = (fread_t)dlsym(RTLD_DEFAULT, "fread"); } - if (fread == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fread" << std::endl; - } if (is_intercepted) { fgetc = (fgetc_t)dlsym(RTLD_NEXT, "fgetc"); } else { fgetc = (fgetc_t)dlsym(RTLD_DEFAULT, "fgetc"); } - if (fgetc == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fgetc" << std::endl; - } if (is_intercepted) { getc = (getc_t)dlsym(RTLD_NEXT, "getc"); } else { getc = (getc_t)dlsym(RTLD_DEFAULT, "getc"); } - if (getc == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "getc" << std::endl; - } if (is_intercepted) { getw = (getw_t)dlsym(RTLD_NEXT, "getw"); } else { getw = (getw_t)dlsym(RTLD_DEFAULT, "getw"); } - if (getw == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "getw" << std::endl; - } if (is_intercepted) { fgets = (fgets_t)dlsym(RTLD_NEXT, "fgets"); } else { fgets = (fgets_t)dlsym(RTLD_DEFAULT, "fgets"); } - if (fgets == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fgets" << std::endl; - } if (is_intercepted) { rewind = (rewind_t)dlsym(RTLD_NEXT, "rewind"); } else { rewind = (rewind_t)dlsym(RTLD_DEFAULT, "rewind"); } - if (rewind == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "rewind" << std::endl; - } if (is_intercepted) { fseek = (fseek_t)dlsym(RTLD_NEXT, "fseek"); } else { fseek = (fseek_t)dlsym(RTLD_DEFAULT, "fseek"); } - if (fseek == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fseek" << std::endl; - } if (is_intercepted) { fseeko = (fseeko_t)dlsym(RTLD_NEXT, "fseeko"); } else { fseeko = (fseeko_t)dlsym(RTLD_DEFAULT, "fseeko"); } - if (fseeko == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fseeko" << std::endl; - } if (is_intercepted) { fseeko64 = (fseeko64_t)dlsym(RTLD_NEXT, "fseeko64"); } else { fseeko64 = (fseeko64_t)dlsym(RTLD_DEFAULT, "fseeko64"); } - if (fseeko64 == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fseeko64" << std::endl; - } if (is_intercepted) { fsetpos = (fsetpos_t)dlsym(RTLD_NEXT, "fsetpos"); } else { fsetpos = (fsetpos_t)dlsym(RTLD_DEFAULT, "fsetpos"); } - if (fsetpos == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fsetpos" << std::endl; - } if (is_intercepted) { fsetpos64 = (fsetpos64_t)dlsym(RTLD_NEXT, "fsetpos64"); } else { fsetpos64 = (fsetpos64_t)dlsym(RTLD_DEFAULT, "fsetpos64"); } - if (fsetpos64 == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "fsetpos64" << std::endl; - } if (is_intercepted) { ftell = (ftell_t)dlsym(RTLD_NEXT, "ftell"); } else { ftell = (ftell_t)dlsym(RTLD_DEFAULT, "ftell"); } - if (ftell == nullptr) { - LOG(FATAL) << "HERMES Adapter failed to map symbol: " - "ftell" << std::endl; - } } }; } // namespace hermes::adapter::stdio From 725133ebb550d9bbeeca4a257e4cfe69929f1062 Mon Sep 17 00:00:00 2001 From: lukemartinlogan Date: Tue, 18 Oct 2022 13:33:32 -0500 Subject: [PATCH 8/8] Restore original interceptor.cc --- adapter/interceptor.cc | 55 ++++++++---------------------------------- adapter/interceptor.h | 35 ++++++++++----------------- ci/lint.sh | 13 +++++----- 3 files changed, 30 insertions(+), 73 deletions(-) diff --git a/adapter/interceptor.cc b/adapter/interceptor.cc index 41180f30b..ca477c476 100644 --- a/adapter/interceptor.cc +++ b/adapter/interceptor.cc @@ -73,8 +73,7 @@ void PopulateBufferingPath() { populated = true; } -bool IsTracked(const std::string& path, bool log) { - if (log) { LOG(INFO) << "IsTracked path? " << path << std::endl; } +bool IsTracked(const std::string& path) { if (hermes::adapter::exit) { return false; } @@ -83,12 +82,7 @@ bool IsTracked(const std::string& path, bool log) { std::string abs_path = WeaklyCanonical(path).string(); for (const auto& pth : INTERCEPTOR_LIST->hermes_flush_exclusion) { - if (abs_path.rfind(pth) != std::string::npos) { - if (log) { - LOG(INFO) << "Path " << path - << " is not tracked (hermes_flush_exclusion)" - << " because: " << pth << std::endl; - } + if (abs_path.find(pth) != std::string::npos) { return false; } } @@ -98,32 +92,20 @@ bool IsTracked(const std::string& path, bool log) { } for (const auto& pth : INTERCEPTOR_LIST->hermes_paths_inclusion) { - if (abs_path.rfind(pth) != std::string::npos) { - if (log) { - LOG(INFO) << "Path " << path << "is tracked (hermes_paths_inclusion)" - << " because: " << pth << std::endl; - } + if (abs_path.find(pth) != std::string::npos) { return true; } } for (auto &pth : kPathExclusions) { - if (abs_path.rfind(pth) != std::string::npos) { - if (log) { - LOG(INFO) << "Path " << path << " is not tracked (kPathExclusions)" - << " because: " << pth << std::endl; - } + if (abs_path.find(pth) != std::string::npos) { return false; } } for (const auto& pth : INTERCEPTOR_LIST->hermes_paths_exclusion) { - if (abs_path.rfind(pth) != std::string::npos) { - if (log) { - LOG(INFO) << "Path " << path - << " is not tracked (hermes_paths_exclusion)" - << " because: " << pth << std::endl; - } + if (abs_path.find(pth) != std::string::npos || + pth.find(abs_path) != std::string::npos) { return false; } } @@ -132,18 +114,10 @@ bool IsTracked(const std::string& path, bool log) { auto buffer_mode = INTERCEPTOR_LIST->adapter_mode; if (buffer_mode == AdapterMode::kBypass) { if (list->adapter_paths.empty()) { - if (log) { - LOG(INFO) << "Path " << path << " is not tracked (kBypass)" - << " because adapter_paths empty " << std::endl; - } return false; } else { for (const auto& pth : list->adapter_paths) { if (path.find(pth) == 0) { - if (log) { - LOG(INFO) << "Path " << path << " is not tracked (kBypass)" - << " because: " << pth << std::endl; - } return false; } } @@ -154,25 +128,16 @@ bool IsTracked(const std::string& path, bool log) { } } -bool IsTracked(FILE* fh, bool log) { - if (log) { LOG(INFO) << "IsTracked fh?" << std::endl; } +bool IsTracked(FILE* fh) { if (hermes::adapter::exit) return false; atexit(OnExit); - std::string path = GetFilenameFromFP(fh); - if (path == "") return false; - return IsTracked(path, log); + return IsTracked(GetFilenameFromFP(fh)); } -bool IsTracked(int fd, bool log) { - if (log) { LOG(INFO) << "IsTracked fd?" << std::endl; } +bool IsTracked(int fd) { if (hermes::adapter::exit) return false; atexit(OnExit); - std::string path = GetFilenameFromFD(fd); - if (path == "") { - if (log) { LOG(INFO) << fd << "is not tracked" << std::endl; } - return false; - } - return IsTracked(path, log); + return IsTracked(GetFilenameFromFD(fd)); } void OnExit(void) { hermes::adapter::exit = true; } diff --git a/adapter/interceptor.h b/adapter/interceptor.h index 8ede96e0c..bc68196a5 100644 --- a/adapter/interceptor.h +++ b/adapter/interceptor.h @@ -55,24 +55,22 @@ inline std::vector StringSplit(const char* str, char delimiter) { } return v; } - +inline std::string GetFilenameFromFP(FILE* fh) { + char proclnk[kMaxPathLen]; + char filename[kMaxPathLen]; + int fno = fileno(fh); + snprintf(proclnk, kMaxPathLen, "/proc/self/fd/%d", fno); + size_t r = readlink(proclnk, filename, kMaxPathLen); + filename[r] = '\0'; + return filename; +} inline std::string GetFilenameFromFD(int fd) { char proclnk[kMaxPathLen]; char filename[kMaxPathLen]; - if (fd == -1) return ""; snprintf(proclnk, kMaxPathLen, "/proc/self/fd/%d", fd); size_t r = readlink(proclnk, filename, kMaxPathLen); - if (r > 0) { - return std::string(filename, r); - } else { - return ""; - } -} - -inline std::string GetFilenameFromFP(FILE* fh) { - if (fh == nullptr) return ""; - int fno = fileno(fh); - return GetFilenameFromFD(fno); + filename[r] = '\0'; + return filename; } /** @@ -202,12 +200,6 @@ void OnExit(void); } \ } -#define REQUIRE_API(api_name) \ - if (real_api->api_name == nullptr) { \ - LOG(FATAL) << "HERMES Adapter failed to map symbol: " \ - #api_name << std::endl; \ - exit(1); - namespace hermes::adapter { /** * Loads the buffering paths for Hermes Adapter to exclude. It inserts the @@ -224,7 +216,7 @@ void PopulateBufferingPath(); * @return true, if file should be tracked. * false, if file should not be intercepted. */ -bool IsTracked(const std::string& path, bool log = false); +bool IsTracked(const std::string& path); /** * Check if fh should be tracked. In this method, first Convert fh to path. @@ -234,8 +226,7 @@ bool IsTracked(const std::string& path, bool log = false); * @return true, if file should be tracked. * false, if file should not be intercepted. */ -bool IsTracked(FILE* fh, bool log = false); -bool IsTracked(int fd, bool log = false); +bool IsTracked(FILE* fh); } // namespace hermes::adapter #else /** diff --git a/ci/lint.sh b/ci/lint.sh index b35b21522..19e5db779 100644 --- a/ci/lint.sh +++ b/ci/lint.sh @@ -1,10 +1,11 @@ +#!/bin/bash HERMES_ROOT=$1 ADAPTER=${HERMES_ROOT}/adapter cpplint --recursive \ ---exclude=${HERMES_ROOT}/src/stb_ds.h \ ---exclude=${ADAPTER}/posix/real_api.h \ ---exclude=${ADAPTER}/stdio/real_api.h \ ---exclude=${ADAPTER}/mpiio/real_api.h \ -${HERMES_ROOT}/adapter ${HERMES_ROOT}/benchmarks ${HERMES_ROOT}/data_stager \ -${HERMES_ROOT}/src ${HERMES_ROOT}/test +--exclude="${HERMES_ROOT}/src/stb_ds.h" \ +--exclude="${ADAPTER}/posix/real_api.h" \ +--exclude="${ADAPTER}/stdio/real_api.h" \ +--exclude="${ADAPTER}/mpiio/real_api.h" \ +"${HERMES_ROOT}/adapter" "${HERMES_ROOT}/benchmarks" "${HERMES_ROOT}/data_stager" \ +"${HERMES_ROOT}/src" "${HERMES_ROOT}/test"