diff --git a/CMake/Testing/Temporary/CTestCostData.txt b/CMake/Testing/Temporary/CTestCostData.txt new file mode 100644 index 000000000..ed97d539c --- /dev/null +++ b/CMake/Testing/Temporary/CTestCostData.txt @@ -0,0 +1 @@ +--- diff --git a/CMake/Testing/Temporary/LastTest.log b/CMake/Testing/Temporary/LastTest.log new file mode 100644 index 000000000..c168870e4 --- /dev/null +++ b/CMake/Testing/Temporary/LastTest.log @@ -0,0 +1,3 @@ +Start testing: Sep 28 12:37 CDT +---------------------------------------------------------- +End testing: Sep 28 12:37 CDT diff --git a/CMakeLists.txt b/CMakeLists.txt index f1f3760b7..49e13dfba 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -399,6 +399,8 @@ if(HERMES_BUILD_BUFFER_POOL_VISUALIZER AND SDL2_FOUND) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/src/buffer_pool_visualizer) endif() +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/data_stager) + #----------------------------------------------------------------------------- # Testing #----------------------------------------------------------------------------- diff --git a/adapter/Testing/Temporary/LastTest.log b/adapter/Testing/Temporary/LastTest.log new file mode 100644 index 000000000..ffe3e1431 --- /dev/null +++ b/adapter/Testing/Temporary/LastTest.log @@ -0,0 +1,3 @@ +Start testing: Sep 28 12:36 CDT +---------------------------------------------------------- +End testing: Sep 28 12:36 CDT diff --git a/adapter/adapter_generator/README.md b/adapter/adapter_generator/README.md new file mode 100644 index 000000000..a43b729a1 --- /dev/null +++ b/adapter/adapter_generator/README.md @@ -0,0 +1,14 @@ +# Adapter Generator + +This library generates an interceptor "API" class used to store the original implementations +of a library for an LD_PRELOAD interceptor to use. + +## Usage + +To create the interceptor for POSIX: +```bash +cd /path/to/adapter_generator +python3 posix.py +``` + +This will create the file adapter/posix/posix.h \ No newline at end of file diff --git a/adapter/adapter_generator/adapter_generator/create_interceptor.py b/adapter/adapter_generator/adapter_generator/create_interceptor.py new file mode 100644 index 000000000..4188cf3b2 --- /dev/null +++ b/adapter/adapter_generator/adapter_generator/create_interceptor.py @@ -0,0 +1,146 @@ +import sys,os +import re + +preamble = """/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */""" + +class Api: + def __init__(self, api_str): + self.decompose_prototype(api_str) + + def _is_text(self, tok): + first_is_text = re.match("[_a-zA-Z]", tok[0]) is not None + if not first_is_text: + return False + return re.match("[_a-zA-Z0-9]+", tok) is not None + + def _clean(self, toks): + return [tok for tok in toks if tok is not None and len(tok) > 0] + + def get_arg_tuple(self, arg): + arg_toks = self._clean(re.split("[ ]|(\*+)", arg)) + if len(arg_toks) == 1: + if arg_toks[0] == '...': + type = "" + name = "..." + return (type, name) + type = " ".join(arg_toks[:-1]) + name = arg_toks[-1] + return (type, name) + + def decompose_prototype(self, api_str): + toks = self._clean(re.split("[()]", api_str)) + proto, args = toks[0], toks[1] + + try: + proto = self._clean(re.split("[ ]|(\*+)", proto)) + self.name = proto[-1] + self.ret = " ".join(proto[:-1]) + self.real_name = self.name + self.type = f"{self.name}_t" + except: + print(f"Failed to decompose proto name: {proto}") + exit() + + try: + self.var_defs = [] + args = args.split(',') + for arg in args: + self.var_defs.append(self.get_arg_tuple(arg)) + except: + print(f"Failed to decompose proto args: {args}") + exit(1) + + def get_args(self): + if len(self.var_defs) == 0: + return "" + try: + args = [" ".join(arg_tuple) for arg_tuple in self.var_defs] + except: + print(f"Failed to get arg list: {self.var_defs}") + exit(1) + return ", ".join(args) + + def pass_args(self): + if self.var_defs is None: + return "" + args = [arg[-1] for arg in self.var_defs] + return ", ".join(args) + +class ApiClass: + def __init__(self, namespace, apis, includes, path=None, do_save=True): + self.apis = apis + self.lines = [] + + self.lines.append(preamble) + self.lines.append("") + self.lines.append(f"#ifndef HERMES_ADAPTER_{namespace.upper()}_H") + self.lines.append(f"#define HERMES_ADAPTER_{namespace.upper()}_H") + + self.lines.append("#include ") + self.lines.append("#include ") + self.lines.append("#include ") + self.lines.append("#include ") + self.lines.append("#include \"interceptor.h\"") + self.lines.append("#include \"filesystem/filesystem.h\"") + for include in includes: + self.lines.append(f"#include {include}") + self.lines.append("") + + self.lines.append(f"namespace hermes::adapter::{namespace} {{") + self.lines.append(f"") + self.lines.append(f"class API {{") + + self.lines.append(f" public:") + for api in self.apis: + self.add_intercept_api(api) + + self.lines.append(f" API() {{") + self.lines.append(f" void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, \"{namespace}_intercepted\");") + for api in self.apis: + self.init_api(api) + self.lines.append(f" }}") + self.lines.append(f"}};") + self.lines.append(f"}} // namespace hermes::adapter::{namespace}") + + self.lines.append("") + self.lines.append(f"#endif // HERMES_ADAPTER_{namespace.upper()}_H") + self.lines.append("") + self.text = "\n".join(self.lines) + + if do_save: + self.save(path, namespace) + else: + print(self.text) + + + def save(self, path, namespace): + if path is None: + ns_dir = os.path.dirname(os.getcwd()) + path = os.path.join(ns_dir, namespace, f"real_api.h") + with open(path, "w") as fp: + fp.write(self.text) + + def add_intercept_api(self, api): + self.lines.append(f" typedef {api.ret} (*{api.type})({api.get_args()});") + self.lines.append(f" {api.ret} (*{api.real_name})({api.get_args()}) = nullptr;") + + def init_api(self, api): + self.lines.append(f" if (is_intercepted) {{") + self.lines.append(f" {api.real_name} = ({api.type})dlsym(RTLD_NEXT, \"{api.name}\");") + self.lines.append(f" }} else {{") + self.lines.append(f" {api.real_name} = ({api.type})dlsym(RTLD_DEFAULT, \"{api.name}\");") + self.lines.append(f" }}") + self.lines.append(f" if ({api.real_name} == nullptr) {{") + self.lines.append(f" LOG(FATAL) << \"HERMES Adapter failed to map symbol: \"") + self.lines.append(f" \"{api.name}\" << std::endl;") + self.lines.append(f" }}") \ No newline at end of file diff --git a/adapter/adapter_generator/mpiio.py b/adapter/adapter_generator/mpiio.py new file mode 100644 index 000000000..8bbaa02b2 --- /dev/null +++ b/adapter/adapter_generator/mpiio.py @@ -0,0 +1,39 @@ +from adapter_generator.create_interceptor import Api,ApiClass + +apis = [ + Api("int MPI_Init(int *argc, char ***argv)"), + Api("int MPI_Finalize(void)"), + Api("int MPI_Wait(MPI_Request *req, MPI_Status *status)"), + Api("int MPI_Waitall(int count, MPI_Request *req, MPI_Status *status)"), + Api("int MPI_File_open(MPI_Comm comm, const char *filename, int amode, MPI_Info info, MPI_File *fh)"), + Api("int MPI_File_close(MPI_File *fh)"), + Api("int MPI_File_seek_shared(MPI_File fh, MPI_Offset offset, int whence)"), + Api("int MPI_File_seek(MPI_File fh, MPI_Offset offset, int whence)"), + Api("int MPI_File_get_position(MPI_File fh, MPI_Offset *offset)"), + Api("int MPI_File_read_all(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"), + Api("int MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"), + Api("int MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"), + Api("int MPI_File_read(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"), + Api("int MPI_File_read_ordered(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"), + Api("int MPI_File_read_shared(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"), + Api("int MPI_File_write_all(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"), + Api("int MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"), + Api("int MPI_File_write_at(MPI_File fh, MPI_Offset offset, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"), + Api("int MPI_File_write(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"), + Api("int MPI_File_write_ordered(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"), + Api("int MPI_File_write_shared(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status)"), + Api("int MPI_File_iread_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)"), + Api("int MPI_File_iread(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)"), + Api("int MPI_File_iread_shared(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request)"), + Api("int MPI_File_iwrite_at(MPI_File fh, MPI_Offset offset, const void *buf, int count, MPI_Datatype datatype, MPI_Request *request)"), + Api("int MPI_File_iwrite(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Request *request)"), + Api("int MPI_File_iwrite_shared(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Request *request)"), + Api("int MPI_File_sync(MPI_File fh)"), +] + +includes = [ + "", + "" +] + +ApiClass("mpiio", apis, includes) \ No newline at end of file diff --git a/adapter/adapter_generator/posix.py b/adapter/adapter_generator/posix.py new file mode 100644 index 000000000..391ebc36d --- /dev/null +++ b/adapter/adapter_generator/posix.py @@ -0,0 +1,24 @@ +from adapter_generator.create_interceptor import Api,ApiClass + +apis = [ + Api("int MPI_Init(int *argc, char ***argv)"), + Api("int MPI_Finalize(void)"), + Api("int open(const char *path, int flags, ...)"), + Api("int open64(const char *path, int flags, ...)"), + Api("int __open_2(const char *path, int oflag)"), + Api("int creat(const char *path, mode_t mode)"), + Api("int creat64(const char *path, mode_t mode)"), + Api("ssize_t read(int fd, void *buf, size_t count)"), + Api("ssize_t write(int fd, const void *buf, size_t count)"), + Api("ssize_t pread(int fd, void *buf, size_t count, off_t offset)"), + Api("ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset)"), + Api("ssize_t pread64(int fd, void *buf, size_t count, off64_t offset)"), + Api("ssize_t pwrite64(int fd, const void *buf, size_t count, off64_t offset)"), + Api("off_t lseek(int fd, off_t offset, int whence)"), + Api("off64_t lseek64(int fd, off64_t offset, int whence)"), + Api("int __fxstat(int version, int fd, struct stat *buf)"), + Api("int fsync(int fd)"), + Api("int close(int fd)"), +] + +ApiClass("posix", apis, []) \ No newline at end of file diff --git a/adapter/adapter_generator/stdio.py b/adapter/adapter_generator/stdio.py new file mode 100644 index 000000000..cc4413fa8 --- /dev/null +++ b/adapter/adapter_generator/stdio.py @@ -0,0 +1,38 @@ +from adapter_generator.create_interceptor import Api,ApiClass + +apis = [ + Api("int MPI_Init(int *argc, char ***argv)"), + Api("int MPI_Finalize(void)"), + Api("FILE *fopen(const char *path, const char *mode)"), + Api("FILE *fopen64(const char *path, const char *mode)"), + Api("FILE *fdopen(int fd, const char *mode)"), + Api("FILE *freopen(const char *path, const char *mode, FILE *stream)"), + Api("FILE *freopen64(const char *path, const char *mode, FILE *stream)"), + Api("int fflush(FILE *fp)"), + Api("int fclose(FILE *fp)"), + Api("size_t fwrite(const void *ptr, size_t size, size_t nmemb, FILE *fp)"), + Api("int fputc(int c, FILE *fp)"), + Api("int fgetpos(FILE *fp, fpos_t *pos)"), + Api("int fgetpos64(FILE *fp, fpos64_t *pos)"), + Api("int putc(int c, FILE *fp)"), + Api("int putw(int w, FILE *fp)"), + Api("int fputs(const char *s, FILE *stream)"), + Api("size_t fread(void *ptr, size_t size, size_t nmemb, FILE *stream)"), + Api("int fgetc(FILE *stream)"), + Api("int getc(FILE *stream)"), + Api("int getw(FILE *stream)"), + Api("char *fgets(char *s, int size, FILE *stream)"), + Api("void rewind(FILE *stream)"), + Api("int fseek(FILE *stream, long offset, int whence)"), + Api("int fseeko(FILE *stream, off_t offset, int whence)"), + Api("int fseeko64(FILE *stream, off64_t offset, int whence)"), + Api("int fsetpos(FILE *stream, const fpos_t *pos)"), + Api("int fsetpos64(FILE *stream, const fpos64_t *pos)"), + Api("long int ftell(FILE *fp)"), +] + +includes = [ + "\"cstdio.h\"" +] + +ApiClass("stdio", apis, includes) \ No newline at end of file diff --git a/adapter/adapter_utils.cc b/adapter/adapter_utils.cc index d0df784f8..c00e28674 100644 --- a/adapter/adapter_utils.cc +++ b/adapter/adapter_utils.cc @@ -11,20 +11,22 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +#include "utils.cc" +#include using hermes::u8; -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; namespace { -const fs::path::value_type dot = '.'; -inline bool is_dot(fs::path::value_type c) { return c == dot; } +const stdfs::path::value_type dot = '.'; +inline bool is_dot(stdfs::path::value_type c) { return c == dot; } -inline bool is_dot(const fs::path& path) { +inline bool is_dot(const stdfs::path& path) { const auto& filename = path.native(); return filename.size() == 1 && is_dot(filename[0]); } -inline bool is_dotdot(const fs::path& path) { +inline bool is_dotdot(const stdfs::path& path) { const auto& filename = path.native(); return filename.size() == 2 && is_dot(filename[0]) && is_dot(filename[1]); } @@ -35,7 +37,7 @@ namespace adapter { // NOTE(chogan): Back port of the C++17 standard fileystem implementation from // gcc 9.1 to support gcc 7 -fs::path LexicallyNormal(fs::path &path) { +stdfs::path LexicallyNormal(stdfs::path &path) { /* C++17 [fs.path.generic] p6 - If the path is empty, stop. @@ -50,7 +52,7 @@ fs::path LexicallyNormal(fs::path &path) { - If the last filename is dot-dot, remove any trailing directory-separator. - If the path is empty, add a dot. */ - fs::path ret; + stdfs::path ret; // If the path is empty, stop. if (path.empty()) { return ret; @@ -101,7 +103,7 @@ fs::path LexicallyNormal(fs::path &path) { } } } else if (is_dot(p)) { - ret /= fs::path(); + ret /= stdfs::path(); } else { ret /= p; } @@ -123,19 +125,19 @@ fs::path LexicallyNormal(fs::path &path) { } // NOTE(chogan): Backported from GCC 9 -fs::path WeaklyCanonical(const fs::path& p) { - fs::path result; - if (fs::exists(fs::status(p))) { - return fs::canonical(p); +stdfs::path WeaklyCanonical(const stdfs::path& p) { + stdfs::path result; + if (stdfs::exists(stdfs::status(p))) { + return stdfs::canonical(p); } - fs::path tmp; + stdfs::path tmp; auto iter = p.begin(), end = p.end(); // find leading elements of p that exist: while (iter != end) { tmp = result / *iter; - if (fs::exists(fs::status(tmp))) { - fs::swap(result, tmp); + if (stdfs::exists(stdfs::status(tmp))) { + stdfs::swap(result, tmp); } else { break; } @@ -143,7 +145,7 @@ fs::path WeaklyCanonical(const fs::path& p) { } // canonicalize: if (!result.empty()) { - result = fs::canonical(result); + result = stdfs::canonical(result); } // append the non-existing elements: while (iter != end) { @@ -154,27 +156,27 @@ fs::path WeaklyCanonical(const fs::path& p) { } // NOTE(chogan): Backported from GCC 9 -fs::path WeaklyCanonical(const fs::path& p, std::error_code& ec) { - fs::path result; - fs::file_status st = fs::status(p, ec); +stdfs::path WeaklyCanonical(const stdfs::path& p, std::error_code& ec) { + stdfs::path result; + stdfs::file_status st = stdfs::status(p, ec); if (exists(st)) { - return fs::canonical(p, ec); - } else if (fs::status_known(st)) { + return stdfs::canonical(p, ec); + } else if (stdfs::status_known(st)) { ec.clear(); } else { return result; } - fs::path tmp; + stdfs::path tmp; auto iter = p.begin(), end = p.end(); // find leading elements of p that exist: while (iter != end) { tmp = result / *iter; - st = fs::status(tmp, ec); + st = stdfs::status(tmp, ec); if (exists(st)) { swap(result, tmp); } else { - if (fs::status_known(st)) { + if (stdfs::status_known(st)) { ec.clear(); } break; @@ -201,8 +203,8 @@ fs::path WeaklyCanonical(const fs::path& p, std::error_code& ec) { void ReadGap(const std::string &filename, size_t seek_offset, u8 *read_ptr, size_t read_size, size_t file_bounds) { - if (fs::exists(filename) && - fs::file_size(filename) >= file_bounds) { + if (stdfs::exists(filename) && + stdfs::file_size(filename) >= file_bounds) { LOG(INFO) << "Blob has a gap in write. Read gap from original file.\n"; INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); int fd = open(filename.c_str(), O_RDONLY); diff --git a/adapter/adapter_utils.h b/adapter/adapter_utils.h index 0f5b24ef8..81025278d 100644 --- a/adapter/adapter_utils.h +++ b/adapter/adapter_utils.h @@ -14,10 +14,10 @@ namespace hermes::adapter { -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; -fs::path WeaklyCanonical(const fs::path& p); -fs::path WeaklyCanonical(const fs::path& p, std::error_code& ec); +stdfs::path WeaklyCanonical(const stdfs::path& p); +stdfs::path WeaklyCanonical(const stdfs::path& p, std::error_code& ec); } // namespace hermes::adapter diff --git a/adapter/filesystem/filesystem.cc b/adapter/filesystem/filesystem.cc new file mode 100644 index 000000000..65977fb30 --- /dev/null +++ b/adapter/filesystem/filesystem.cc @@ -0,0 +1,889 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "filesystem.h" +#include "constants.h" +#include "singleton.h" +#include "mapper/mapper_factory.h" +#include "interceptor.h" +#include "metadata_manager.h" +#include "vbucket.h" +#include "adapter_utils.cc" + +#include +#include + +namespace stdfs = std::experimental::filesystem; + +namespace hermes::adapter::fs { + +static bool IsAsyncFlush(const std::string &path_str) { + bool result = (INTERCEPTOR_LIST->Persists(path_str) && + global_flushing_mode == FlushingMode::kAsynchronous); + return result; +} + +File Filesystem::Open(AdapterStat &stat, const std::string &path) { + std::string path_str = WeaklyCanonical(path).string(); + File f = _RealOpen(stat, path); + if (!f.status_) { return f; } + Open(stat, f, path); + return f; +} + +void Filesystem::Open(AdapterStat &stat, File &f, const std::string &path) { + std::string path_str = WeaklyCanonical(path).string(); + _InitFile(f); + auto mdm = Singleton::GetInstance(); + auto existing = mdm->Find(f); + int rank = 0; + if (mdm->is_mpi) { + MPI_Comm_rank(stat.comm, &rank); + } + if (!existing.second) { + LOG(INFO) << "File not opened before by adapter" << std::endl; + stat.ref_count = 1; + struct timespec ts; + timespec_get(&ts, TIME_UTC); + stat.st_atim = ts; + stat.st_mtim = ts; + stat.st_ctim = ts; + + /* FIXME(hari) check if this initialization is correct. */ + mdm->InitializeHermes(); + + bool bucket_exists = mdm->GetHermes()->BucketExists(path_str); + + // TODO(llogan): needed for locking in parallel + if (mdm->is_mpi) { + stat.main_lock_blob = "#main_lock"; + if (rank == 0) { + u8 c; + stat.st_bkid = + std::make_shared(path_str, mdm->GetHermes()); + stat.st_bkid->Put(stat.main_lock_blob, &c, 1); + } + MPI_Barrier(stat.comm); + if (rank != 0) { + stat.st_bkid = + std::make_shared(path_str, mdm->GetHermes()); + } + } else { + stat.st_bkid = + std::make_shared(path_str, mdm->GetHermes()); + } + + if (IsAsyncFlush(path_str)) { + // TODO(llogan): should we use a vbucket per-rank instead? + std::string vbucket_name = path_str + "#" + std::to_string(rank); + stat.st_vbkt = + std::make_shared(vbucket_name, mdm->GetHermes()); + auto offset_map = std::unordered_map(); + stat.st_persist = + std::make_shared(path_str, offset_map, false); + stat.st_vbkt->Attach(stat.st_persist.get()); + } + _OpenInitStats(f, stat, bucket_exists); + mdm->Create(f, stat); + } else { + LOG(INFO) << "File opened before by adapter" << std::endl; + existing.first.ref_count++; + struct timespec ts; + timespec_get(&ts, TIME_UTC); + existing.first.st_atim = ts; + existing.first.st_ctim = ts; + mdm->Update(f, existing.first); + } +} + +void Filesystem::_PutWithFallback(AdapterStat &stat, + const std::string &blob_name, + const std::string &filename, + u8 *data, size_t size, + size_t offset, + IoStatus &io_status, IoOptions &opts) { + hapi::Context ctx; + const char *hermes_write_only = getenv(kHermesWriteOnlyVar); + if (hermes_write_only && hermes_write_only[0] == '1') { + // Custom DPE for write-only apps like VPIC + ctx.rr_retry = true; + ctx.disable_swap = true; + } + hapi::Status put_status = stat.st_bkid->Put(blob_name, data, size, ctx); + if (put_status.Failed()) { + if (opts.with_fallback_) { + LOG(WARNING) << "Failed to Put Blob " << blob_name << " to Bucket " + << filename << ". Falling back to posix I/O." << std::endl; + _RealWrite(filename, offset, size, data, io_status, opts); + } + } else { + stat.st_blobs.emplace(blob_name); + } +} + +size_t Filesystem::Write(File &f, AdapterStat &stat, const void *ptr, + size_t off, size_t total_size, + IoStatus &io_status, IoOptions opts) { + (void) f; + std::shared_ptr &bkt = stat.st_bkid; + std::string filename = bkt->GetName(); + LOG(INFO) << "Write called for filename: " << filename << " on offset: " + << stat.st_ptr << " and size: " << total_size << std::endl; + + size_t ret; + auto mdm = Singleton::GetInstance(); + BlobPlacements mapping; + auto mapper = MapperFactory().Get(kMapperType); + mapper->map(off, total_size, mapping); + size_t data_offset = 0; + + for (const auto &p : mapping) { + BlobPlacementIter wi(f, stat, filename, p, bkt, io_status, opts); + wi.blob_name_ = wi.p_.CreateBlobName(); + wi.blob_exists_ = wi.bkt_->ContainsBlob(wi.blob_name_); + wi.blob_start_ = p.page_ * kPageSize; + wi.mem_ptr_ = (u8 *)ptr + data_offset; + if (p.blob_size_ != kPageSize && opts.coordinate_ + && stat.main_lock_blob.size() > 0) { + _CoordinatedPut(wi); + } else { + _UncoordinatedPut(wi); + } + data_offset += p.blob_size_; + } + off_t f_offset = off + data_offset; + if (opts.seek_) { stat.st_ptr = f_offset; } + stat.st_size = std::max(stat.st_size, f_offset); + + struct timespec ts; + timespec_get(&ts, TIME_UTC); + stat.st_mtim = ts; + stat.st_ctim = ts; + + mdm->Update(f, stat); + ret = data_offset; + _IoStats(data_offset, io_status, opts); + return ret; +} + +bool Lock(const std::string &bucket, const std::string &blob_name) { + auto mdm = Singleton::GetInstance(); + auto &hermes = mdm->GetHermes(); + SharedMemoryContext *context = &hermes->context_; + RpcContext *rpc = &hermes->rpc_; + BucketID bucket_id = GetBucketId(context, rpc, bucket.c_str()); + BlobID lock_id = GetBlobId(context, rpc, blob_name, bucket_id, true); + bool ret = LockBlob(context, rpc, lock_id); + return ret; +} + +void Unlock(const std::string &bucket, const std::string &blob_name) { + auto mdm = Singleton::GetInstance(); + auto &hermes = mdm->GetHermes(); + SharedMemoryContext *context = &hermes->context_; + RpcContext *rpc = &hermes->rpc_; + BucketID bucket_id = GetBucketId(context, rpc, bucket.c_str()); + BlobID lock_id = GetBlobId(context, rpc, blob_name, bucket_id, true); + UnlockBlob(context, rpc, lock_id); +} + +void Filesystem::_CoordinatedPut(BlobPlacementIter &wi) { + LOG(INFO) << "Starting coordinate PUT" + << " blob: " << wi.p_.page_ + << " off: " << wi.p_.blob_off_ + << " size: " << wi.p_.blob_size_ + << " pid: " << getpid() << std::endl; + + // TODO(llogan): make put async, instead of doing a lock like this + Lock(wi.filename_, wi.stat_.main_lock_blob); + LOG(INFO) << "Acquire lock: " << wi.stat_.main_lock_blob << + " for process: " << getpid() << std::endl; + wi.blob_exists_ = wi.bkt_->ContainsBlob(wi.blob_name_); + _UncoordinatedPut(wi); + LOG(INFO) << "Unlocking for process: " << getpid() << std::endl; + Unlock(wi.filename_, wi.stat_.main_lock_blob); +} + +void Filesystem::_UncoordinatedPut(BlobPlacementIter &wi) { + LOG(INFO) << "Starting uncoordinate PUT" + << " blob: " << wi.p_.page_ + << " off: " << wi.p_.blob_off_ + << " size: " << wi.p_.blob_size_ + << " pid: " << getpid() << std::endl; + if (wi.blob_exists_) { + if (wi.p_.blob_off_ == 0) { + _WriteToExistingAligned(wi); + } else { + _WriteToExistingUnaligned(wi); + } + } else { + if (wi.p_.blob_off_ == 0) { + _WriteToNewAligned(wi); + } else { + _WriteToNewUnaligned(wi); + } + } + if (IsAsyncFlush(wi.filename_)) { + hapi::Trait *trait = wi.stat_.st_vbkt->GetTrait(hapi::TraitType::PERSIST); + if (trait) { + hapi::PersistTrait *persist_trait = (hapi::PersistTrait *)trait; + persist_trait->offset_map.emplace(wi.blob_name_, wi.blob_start_); + } + wi.stat_.st_vbkt->Link(wi.blob_name_, wi.filename_); + } +} + +void Filesystem::_WriteToNewAligned(BlobPlacementIter &wi) { + LOG(INFO) << "Create new blob (aligned)" + << " offset: " << wi.p_.blob_off_ + << " size: " << wi.p_.blob_size_ << std::endl; + _PutWithFallback(wi.stat_, wi.blob_name_, wi.filename_, wi.mem_ptr_, + wi.p_.blob_size_, wi.p_.bucket_off_, + wi.io_status_, wi.opts_); +} + +void Filesystem::_WriteToNewUnaligned(BlobPlacementIter &wi) { + LOG(INFO) << "Create new blob (unaligned)" + << " offset: " << wi.p_.blob_off_ + << " size: " << wi.p_.blob_size_ << std::endl; + hapi::Blob final_data(wi.p_.blob_off_ + wi.p_.blob_size_); + IoOptions opts = IoOptions::DirectIo(wi.opts_); + Read(wi.f_, wi.stat_, final_data.data(), wi.blob_start_, + wi.p_.blob_off_, wi.io_status_, opts); + memcpy(final_data.data() + wi.p_.blob_off_, wi.mem_ptr_, + wi.p_.blob_size_); + _PutWithFallback(wi.stat_, wi.blob_name_, wi.filename_, + final_data.data(), final_data.size(), wi.blob_start_, + wi.io_status_, wi.opts_); +} + +void Filesystem::_WriteToExistingAligned(BlobPlacementIter &wi) { + LOG(INFO) << "Modify existing blob (aligned)" + << " offset: " << wi.p_.blob_off_ + << " size: " << wi.p_.blob_size_ << std::endl; + + hapi::Blob temp(0); + auto existing_blob_size = wi.bkt_->Get(wi.blob_name_, temp); + if (wi.p_.blob_size_ >= existing_blob_size) { + LOG(INFO) << "Overwrite blob " << wi.blob_name_ + << " of size:" << wi.p_.blob_size_ << "." << std::endl; + _PutWithFallback(wi.stat_, wi.blob_name_, wi.filename_, + wi.mem_ptr_, wi.p_.blob_size_, wi.blob_start_, + wi.io_status_, wi.opts_); + } else { + LOG(INFO) << "Update blob " << wi.blob_name_ + << " of size:" << existing_blob_size << "." << std::endl; + hapi::Blob existing_data(existing_blob_size); + wi.bkt_->Get(wi.blob_name_, existing_data); + memcpy(existing_data.data(), wi.mem_ptr_, wi.p_.blob_size_); + _PutWithFallback(wi.stat_, wi.blob_name_, wi.filename_, + existing_data.data(), existing_data.size(), + wi.blob_start_, wi.io_status_, wi.opts_); + } +} + +void Filesystem::_WriteToExistingUnaligned(BlobPlacementIter &wi) { + LOG(INFO) << "Modify existing blob (unaligned)" + << " offset: " << wi.p_.blob_off_ + << " size: " << wi.p_.blob_size_ << std::endl; + + auto new_size = wi.p_.blob_off_ + wi.p_.blob_size_; + hapi::Blob temp(0); + auto existing_blob_size = wi.bkt_->Get(wi.blob_name_, temp); + hapi::Blob existing_data(existing_blob_size); + wi.bkt_->Get(wi.blob_name_, existing_data); + wi.bkt_->DeleteBlob(wi.blob_name_); + if (new_size < existing_blob_size) { + new_size = existing_blob_size; + } + hapi::Blob final_data(new_size); + + // [0, existing_data) + memcpy(final_data.data(), existing_data.data(), + existing_data.size()); + + // [existing_data, blob_off) + if (existing_data.size() < wi.p_.blob_off_) { + IoOptions opts = IoOptions::DirectIo(wi.opts_); + Read(wi.f_, wi.stat_, + final_data.data() + existing_data.size(), + wi.blob_start_ + existing_data.size(), + wi.p_.blob_off_ - existing_data.size(), + wi.io_status_, + opts); + } + + // [blob_off, blob_off + blob_size) + memcpy(final_data.data() + wi.p_.blob_off_, wi.mem_ptr_, + wi.p_.blob_size_); + + // [blob_off + blob_size, existing_blob_size) + if (existing_blob_size > wi.p_.blob_off_ + wi.p_.blob_size_) { + LOG(INFO) << "Retain last portion of blob as Blob is bigger than the " + "update." << std::endl; + auto off_t = wi.p_.blob_off_ + wi.p_.blob_size_; + memcpy(final_data.data() + off_t, existing_data.data() + off_t, + existing_blob_size - off_t); + } + + // Store updated blob + _PutWithFallback(wi.stat_, wi.blob_name_, wi.filename_, + final_data.data(), + final_data.size(), + wi.blob_start_, + wi.io_status_, + wi.opts_); +} + +size_t Filesystem::Read(File &f, AdapterStat &stat, void *ptr, + size_t off, size_t total_size, + IoStatus &io_status, IoOptions opts) { + (void) f; + std::shared_ptr &bkt = stat.st_bkid; + LOG(INFO) << "Read called for filename: " << bkt->GetName() + << " (fd: " << f.fd_ << ")" + << " on offset: " << off + << " and size: " << total_size + << " (stored file size: " << stat.st_size + << " true file size: " << stdfs::file_size(bkt->GetName()) + << ")" << std::endl; + if (stat.st_ptr >= stat.st_size) { + LOG(INFO) << "The current offset: " << stat.st_ptr << + " is larger than file size: " << stat.st_size << std::endl; + return 0; + } + size_t ret; + BlobPlacements mapping; + auto mdm = Singleton::GetInstance(); + auto mapper = MapperFactory().Get(kMapperType); + mapper->map(off, total_size, mapping); + + size_t data_offset = 0; + auto filename = bkt->GetName(); + LOG(INFO) << "Mapping for read has " << mapping.size() << " mapping." + << std::endl; + for (const auto &p : mapping) { + BlobPlacementIter ri(f, stat, filename, p, bkt, io_status, opts); + ri.blob_name_ = ri.p_.CreateBlobName(); + ri.blob_exists_ = bkt->ContainsBlob(ri.blob_name_); + ri.blob_start_ = ri.p_.page_ * kPageSize; + ri.mem_ptr_ = (u8 *)ptr + data_offset; + size_t read_size; + if (ri.blob_exists_) { + size_t min_blob_size = p.blob_off_ + p.blob_size_; + auto existing_blob_size = bkt->Get( + ri.blob_name_, ri.blob_, ri.ctx_); + ri.blob_.resize(existing_blob_size); + if (existing_blob_size >= min_blob_size) { + read_size = _ReadExistingContained(ri); + } else { + read_size = _ReadExistingPartial(ri); + } + } else { + read_size = _ReadNew(ri); + } + data_offset += read_size; + } + if (opts.seek_) { stat.st_ptr += data_offset; } + struct timespec ts; + timespec_get(&ts, TIME_UTC); + stat.st_atim = ts; + stat.st_ctim = ts; + ret = data_offset; + mdm->Update(f, stat); + _IoStats(data_offset, io_status, opts); + return ret; +} + +size_t Filesystem::_ReadExistingContained(BlobPlacementIter &ri) { + LOG(INFO) << "Blob exists and need to read from Hermes from blob: " + << ri.blob_name_ << "." << std::endl; + LOG(INFO) << "Blob have data and need to read from hemes " + "blob: " + << ri.blob_name_ << " offset:" << ri.p_.blob_off_ + << " size:" << ri.p_.blob_size_ << "." << std::endl; + + ri.bkt_->Get(ri.blob_name_, ri.blob_, ri.ctx_); + memcpy(ri.mem_ptr_, ri.blob_.data() + ri.p_.blob_off_, ri.p_.blob_size_); + return ri.p_.blob_size_; +} + +size_t Filesystem::_ReadExistingPartial(BlobPlacementIter &ri) { + LOG(INFO) << "Blob exists and need to partially read from Hermes from blob: " + << ri.blob_name_ << "." << std::endl; + /*if (!stdfs::exists(filename) || + stdfs::file_size(filename) < p.bucket_off_ + p.blob_off_ + p.blob_size_) { + return 0; + }*/ + size_t min_blob_size = ri.p_.blob_off_ + ri.p_.blob_size_; + size_t existing_size = ri.blob_.size(); + size_t new_blob_size = std::max(min_blob_size, existing_size); + size_t bytes_to_read = min_blob_size - existing_size; + ri.blob_.resize(new_blob_size); + + LOG(INFO) << "Blob does not have data and need to read from original " + "filename: " + << ri.filename_ << " offset:" << ri.blob_start_ + existing_size + << " size:" << bytes_to_read << "." + << std::endl; + + size_t ret = _RealRead(ri.filename_, + ri.blob_start_ + existing_size, + bytes_to_read, + ri.blob_.data() + existing_size, + ri.io_status_, + ri.opts_); + + if (ri.opts_.dpe_ != PlacementPolicy::kNone) { + IoOptions opts(ri.opts_); + opts.seek_ = false; + opts.with_fallback_ = false; + Write(ri.f_, ri.stat_, + ri.blob_.data() + ri.p_.blob_off_, + ri.p_.bucket_off_, + new_blob_size, ri.io_status_, opts); + } + + if (ret != bytes_to_read) { + LOG(FATAL) << "Was not able to read all data from the file" << std::endl; + } + + memcpy(ri.mem_ptr_, ri.blob_.data() + ri.p_.blob_off_, ri.p_.blob_size_); + return ri.p_.blob_size_; +} + +size_t Filesystem::_ReadNew(BlobPlacementIter &ri) { + LOG(INFO) + << "Blob does not exists and need to read from original filename: " + << ri.filename_ << " offset:" << ri.p_.bucket_off_ + << " size:" << ri.p_.blob_size_ << "." << std::endl; + + /*if (!stdfs::exists(filename) || + stdfs::file_size(filename) < p.bucket_off_ + p.blob_size_) { + return 0; + }*/ + + auto new_blob_size = ri.p_.blob_off_ + ri.p_.blob_size_; + ri.blob_.resize(new_blob_size); + size_t ret = _RealRead(ri.filename_, ri.blob_start_, + new_blob_size, ri.blob_.data(), + ri.io_status_, ri.opts_); + if (ret != new_blob_size) { + LOG(FATAL) << "Was not able to read full content" << std::endl; + } + + if (ri.opts_.dpe_ != PlacementPolicy::kNone) { + LOG(INFO) << "Placing the read blob in the hierarchy" << std::endl; + IoOptions opts(ri.opts_); + opts.seek_ = false; + opts.with_fallback_ = false; + Write(ri.f_, ri.stat_, + ri.blob_.data() + ri.p_.blob_off_, + ri.p_.bucket_off_, + new_blob_size, ri.io_status_, opts); + } + + memcpy(ri.mem_ptr_, ri.blob_.data() + ri.p_.blob_off_, ri.p_.blob_size_); + return ri.p_.blob_size_; +} + +HermesRequest* Filesystem::AWrite(File &f, AdapterStat &stat, const void *ptr, + size_t off, size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts) { + (void) io_status; + LOG(INFO) << "Starting an asynchronous write" << std::endl; + auto pool = + Singleton::GetInstance(kNumThreads); + HermesRequest *hreq = new HermesRequest(); + auto lambda = + [](Filesystem *fs, File &f, AdapterStat &stat, const void *ptr, + size_t off, size_t total_size, IoStatus &io_status, IoOptions opts) { + return fs->Write(f, stat, ptr, off, total_size, io_status, opts); + }; + auto func = std::bind(lambda, this, f, stat, ptr, off, + total_size, hreq->io_status, opts); + hreq->return_future = pool->run(func); + auto mdm = Singleton::GetInstance(); + mdm->request_map.emplace(req_id, hreq); + return hreq; +} + +HermesRequest* Filesystem::ARead(File &f, AdapterStat &stat, void *ptr, + size_t off, size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts) { + (void) io_status; + auto pool = + Singleton::GetInstance(kNumThreads); + HermesRequest *hreq = new HermesRequest(); + auto lambda = + [](Filesystem *fs, File &f, AdapterStat &stat, void *ptr, + size_t off, size_t total_size, IoStatus &io_status, IoOptions opts) { + return fs->Read(f, stat, ptr, off, total_size, io_status, opts); + }; + auto func = std::bind(lambda, this, f, stat, + ptr, off, total_size, hreq->io_status, opts); + hreq->return_future = pool->run(func); + auto mdm = Singleton::GetInstance(); + mdm->request_map.emplace(req_id, hreq); + return hreq; +} + +size_t Filesystem::Wait(size_t req_id) { + auto mdm = Singleton::GetInstance(); + auto req_iter = mdm->request_map.find(req_id); + if (req_iter == mdm->request_map.end()) { + return 0; + } + HermesRequest *req = (*req_iter).second; + size_t ret = req->return_future.get(); + delete req; + return ret; +} + +void Filesystem::Wait(std::vector &req_ids, + std::vector &ret) { + for (auto &req_id : req_ids) { + ret.emplace_back(Wait(req_id)); + } +} + +off_t Filesystem::Seek(File &f, AdapterStat &stat, + SeekMode whence, off_t offset) { + if (stat.is_append) { + LOG(INFO) + << "File pointer not updating as file was opened in append mode." + << std::endl; + return -1; + } + auto mdm = Singleton::GetInstance(); + switch (whence) { + case SeekMode::kSet: { + stat.st_ptr = offset; + break; + } + case SeekMode::kCurrent: { + stat.st_ptr += offset; + break; + } + case SeekMode::kEnd: { + stat.st_ptr = stat.st_size + offset; + break; + } + default: { + // TODO(hari): throw not implemented error. + } + } + mdm->Update(f, stat); + return stat.st_ptr; +} + +off_t Filesystem::Tell(File &f, AdapterStat &stat) { + (void) f; + return stat.st_ptr; +} + +int Filesystem::Sync(File &f, AdapterStat &stat) { + int rank = 0; + auto mdm = Singleton::GetInstance(); + if (stat.ref_count != 1) { + LOG(INFO) << "File handler is opened by more than one fopen." + << std::endl; + struct timespec ts; + timespec_get(&ts, TIME_UTC); + stat.st_atim = ts; + stat.st_ctim = ts; + mdm->Update(f, stat); + return 0; + } + if (mdm->is_mpi) { + MPI_Comm_rank(stat.comm, &rank); + MPI_Barrier(stat.comm); + } + + // Wait for all async requests to complete + for (auto &req : mdm->request_map) { + Wait(req.first); + } + + hapi::Context ctx; + auto filename = stat.st_bkid->GetName(); + auto persist = INTERCEPTOR_LIST->Persists(filename); + const auto &blob_names = stat.st_blobs; + if (blob_names.empty() || !persist) { + return 0; + } + if (IsAsyncFlush(filename)) { + stat.st_vbkt->WaitForBackgroundFlush(); + return 0; + } + + LOG(INFO) << "Filesystem Sync flushes " << blob_names.size() + << " blobs to filename:" << filename << "." << std::endl; + INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); + + std::string vbucket_name = filename + "#" + + std::to_string(rank) + "#sync"; + hapi::VBucket file_vbucket(vbucket_name, mdm->GetHermes(), ctx); + auto offset_map = std::unordered_map(); + for (const auto &blob_name : blob_names) { + auto status = file_vbucket.Link(blob_name, filename, ctx); + if (!status.Failed()) { + BlobPlacement p; + p.DecodeBlobName(blob_name); + offset_map.emplace(blob_name, p.page_ * kPageSize); + } + } + bool flush_synchronously = true; + hapi::PersistTrait persist_trait(filename, offset_map, + flush_synchronously); + file_vbucket.Attach(&persist_trait, ctx); + file_vbucket.Destroy(ctx); + stat.st_blobs.clear(); + INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename); + mdm->Update(f, stat); + return _RealSync(f); +} + +int Filesystem::Close(File &f, AdapterStat &stat, bool destroy) { + hapi::Context ctx; + int rank = 0; + auto mdm = Singleton::GetInstance(); + if (stat.ref_count > 1) { + LOG(INFO) << "File handler is opened by more than one fopen." + << std::endl; + stat.ref_count--; + struct timespec ts; + timespec_get(&ts, TIME_UTC); + stat.st_atim = ts; + stat.st_ctim = ts; + stat.st_bkid->Release(ctx); + if (stat.st_vbkt) { stat.st_vbkt->Release(); } + mdm->Update(f, stat); + return 0; + } + if (mdm->is_mpi) { + MPI_Comm_rank(stat.comm, &rank); + if (rank != 0) { + stat.st_bkid->Release(ctx); + } + MPI_Barrier(stat.comm); + } + Sync(f, stat); + auto filename = stat.st_bkid->GetName(); + if (mdm->is_mpi) { MPI_Barrier(stat.comm); } + if (IsAsyncFlush(filename)) { + stat.st_vbkt->Destroy(); + } + mdm->Delete(f); + if (mdm->is_mpi) { MPI_Barrier(stat.comm); } + if (destroy) { stat.st_bkid->Destroy(ctx); } + if (stat.amode & MPI_MODE_DELETE_ON_CLOSE) { + stdfs::remove(filename); + } + mdm->FinalizeHermes(); + return _RealClose(f); +} + + +/** + * Variants of Read and Write which do not take an offset as + * input. + * */ + +size_t Filesystem::Write(File &f, AdapterStat &stat, const void *ptr, + size_t total_size, IoStatus &io_status, + IoOptions opts) { + off_t off = Tell(f, stat); + return Write(f, stat, ptr, off, total_size, io_status, opts); +} + +size_t Filesystem::Read(File &f, AdapterStat &stat, void *ptr, + size_t total_size, + IoStatus &io_status, IoOptions opts) { + off_t off = Tell(f, stat); + return Read(f, stat, ptr, off, total_size, io_status, opts); +} + +HermesRequest* Filesystem::AWrite(File &f, AdapterStat &stat, const void *ptr, + size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts) { + off_t off = Tell(f, stat); + return AWrite(f, stat, ptr, off, total_size, req_id, io_status, opts); +} + +HermesRequest* Filesystem::ARead(File &f, AdapterStat &stat, void *ptr, + size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts) { + off_t off = Tell(f, stat); + return ARead(f, stat, ptr, off, total_size, req_id, io_status, opts); +} + + +/** + * Variants of the above functions which retrieve the AdapterStat + * data structure internally. + * */ + +size_t Filesystem::Write(File &f, bool &stat_exists, const void *ptr, + size_t total_size, + IoStatus &io_status, IoOptions opts) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return 0; + } + stat_exists = true; + return Write(f, stat, ptr, total_size, io_status, opts); +} + +size_t Filesystem::Read(File &f, bool &stat_exists, void *ptr, + size_t total_size, + IoStatus &io_status, IoOptions opts) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return 0; + } + stat_exists = true; + return Read(f, stat, ptr, total_size, io_status, opts); +} + +size_t Filesystem::Write(File &f, bool &stat_exists, const void *ptr, + size_t off, size_t total_size, + IoStatus &io_status, IoOptions opts) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return 0; + } + stat_exists = true; + opts.seek_ = false; + return Write(f, stat, ptr, off, total_size, io_status, opts); +} + +size_t Filesystem::Read(File &f, bool &stat_exists, void *ptr, + size_t off, size_t total_size, + IoStatus &io_status, IoOptions opts) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return 0; + } + stat_exists = true; + opts.seek_ = false; + return Read(f, stat, ptr, off, total_size, io_status, opts); +} + +HermesRequest* Filesystem::AWrite(File &f, bool &stat_exists, const void *ptr, + size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return 0; + } + stat_exists = true; + return AWrite(f, stat, ptr, total_size, req_id, io_status, opts); +} + +HermesRequest* Filesystem::ARead(File &f, bool &stat_exists, void *ptr, + size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return 0; + } + stat_exists = true; + return ARead(f, stat, ptr, total_size, req_id, io_status, opts); +} + +HermesRequest* Filesystem::AWrite(File &f, bool &stat_exists, const void *ptr, + size_t off, size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return 0; + } + stat_exists = true; + opts.seek_ = false; + return AWrite(f, stat, ptr, off, total_size, req_id, io_status, opts); +} + +HermesRequest* Filesystem::ARead(File &f, bool &stat_exists, void *ptr, + size_t off, size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return 0; + } + stat_exists = true; + opts.seek_ = false; + return ARead(f, stat, ptr, off, total_size, req_id, io_status, opts); +} + +off_t Filesystem::Seek(File &f, bool &stat_exists, + SeekMode whence, off_t offset) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + return Seek(f, stat, whence, offset); +} + +off_t Filesystem::Tell(File &f, bool &stat_exists) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + return Tell(f, stat); +} + +int Filesystem::Sync(File &f, bool &stat_exists) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + return Sync(f, stat); +} + +int Filesystem::Close(File &f, bool &stat_exists, bool destroy) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + return Close(f, stat, destroy); +} + +} // namespace hermes::adapter::fs diff --git a/adapter/filesystem/filesystem.h b/adapter/filesystem/filesystem.h new file mode 100644 index 000000000..6b04825c9 --- /dev/null +++ b/adapter/filesystem/filesystem.h @@ -0,0 +1,356 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_ADAPTER_FILESYSTEM_FILESYSTEM_H_ +#define HERMES_ADAPTER_FILESYSTEM_FILESYSTEM_H_ + +#include "enumerations.h" +#include +#include +#include +#include +#include +#include +#include +#include "mapper/mapper_factory.h" +#include +#include + +namespace hapi = hermes::api; + +namespace hermes::adapter::fs { + +const char kStringDelimiter = '#'; +const MapperType kMapperType = MapperType::BALANCED; +FlushingMode global_flushing_mode; +const int kNumThreads = 1; + +enum class SeekMode { + kNone = -1, + kSet = SEEK_SET, + kCurrent = SEEK_CUR, + kEnd = SEEK_END +}; + +struct AdapterStat { + std::shared_ptr st_bkid; /* bucket associated with the file */ + /** VBucket for persisting data asynchronously. */ + std::shared_ptr st_vbkt; + /** Blob for locking when coordination is needed. */ + std::string main_lock_blob; + /** Used for async flushing. */ + std::shared_ptr st_persist; + std::set + st_blobs; /* Blobs access in the bucket */ + i32 ref_count; /* # of time process opens a file */ + int flags; /* open() flags for POSIX */ + mode_t st_mode; /* protection */ + uid_t st_uid; /* user ID of owner */ + gid_t st_gid; /* group ID of owner */ + off_t st_size; /* total size, in bytes */ + off_t st_ptr; /* Current ptr of FILE */ + blksize_t st_blksize; /* blocksize for blob within bucket */ + timespec st_atim; /* time of last access */ + timespec st_mtim; /* time of last modification */ + timespec st_ctim; /* time of last status change */ + std::string mode_str; /* mode used for fopen() */ + + bool is_append; /* File is in append mode */ + int amode; /* access mode */ + MPI_Info info; /* Info object (handle) */ + MPI_Comm comm; /* Communicator for the file.*/ + bool atomicity; /* Consistency semantics for data-access */ + + AdapterStat() + : st_bkid(), + st_blobs(CompareBlobs), + ref_count(1), + flags(0), + st_mode(), + st_uid(), + st_gid(), + st_size(0), + st_ptr(0), + st_blksize(4096), + st_atim(), + st_mtim(), + st_ctim(), + is_append(false), + amode(0), + comm(MPI_COMM_SELF), + atomicity(false) {} + + static bool CompareBlobs(const std::string &a, const std::string &b) { + return std::stol(a) < std::stol(b); + } +}; + +struct File { + int fd_; + FILE *fh_; + MPI_File mpi_fh_; + + dev_t st_dev; + ino_t st_ino; + bool status_; + int mpi_status_; + + File() : fd_(-1), + fh_(nullptr), + mpi_fh_(nullptr), + st_dev(-1), + st_ino(-1), + status_(true), + mpi_status_(MPI_SUCCESS) {} + + File(const File &old) { + Copy(old); + } + + File &operator=(const File &old) { + Copy(old); + return *this; + } + + void Copy(const File &old) { + fd_ = old.fd_; + fh_ = old.fh_; + mpi_fh_ = old.mpi_fh_; + st_dev = old.st_dev; + st_ino = old.st_ino; + status_ = old.status_; + } + + bool operator==(const File &old) const { + return (st_dev == old.st_dev) && (st_ino == old.st_ino) && + (mpi_fh_ == old.mpi_fh_); + } + + std::size_t hash() const { + std::size_t result; + std::size_t h1 = std::hash{}(st_dev); + std::size_t h2 = std::hash{}(st_ino); + std::size_t h3 = std::hash{}(mpi_fh_); + result = h1 ^ h2 ^ h3; + return result; + } +}; + +struct IoOptions { + PlacementPolicy dpe_; + bool coordinate_; + bool seek_; + bool with_fallback_; + MPI_Datatype mpi_type_; + int count_; + IoOptions() : + dpe_(PlacementPolicy::kNone), + coordinate_(true), + seek_(true), + with_fallback_(true), + mpi_type_(MPI_CHAR), count_(0) {} + + static IoOptions WithParallelDpe(PlacementPolicy dpe) { + IoOptions opts; + opts.dpe_ = dpe; + opts.coordinate_ = true; + return opts; + } + + static IoOptions DirectIo(IoOptions &cur_opts) { + IoOptions opts(cur_opts); + opts.seek_ = false; + opts.dpe_ = PlacementPolicy::kNone; + opts.with_fallback_ = true; + return opts; + } + + static IoOptions DataType(MPI_Datatype mpi_type, bool seek = true) { + IoOptions opts; + opts.mpi_type_ = mpi_type; + opts.seek_ = seek; + return opts; + } +}; + +struct IoStatus { + int mpi_ret_; + MPI_Status mpi_status_; + MPI_Status *mpi_status_ptr_; + + IoStatus() : mpi_ret_(MPI_SUCCESS), mpi_status_ptr_(&mpi_status_) {} +}; + +struct HermesRequest { + std::future return_future; + IoStatus io_status; +}; + +struct BlobPlacementIter { + File &f_; + AdapterStat &stat_; + const std::string &filename_; + const BlobPlacement &p_; + std::shared_ptr &bkt_; + IoStatus &io_status_; + IoOptions &opts_; + + std::string blob_name_; + u8 *mem_ptr_; + size_t blob_start_; + hapi::Context ctx_; + hapi::Blob blob_; + int rank_; + int nprocs_; + bool blob_exists_; + + explicit BlobPlacementIter(File &f, AdapterStat &stat, + const std::string &filename, + const BlobPlacement &p, + std::shared_ptr &bkt, + IoStatus &io_status, + IoOptions &opts) : + f_(f), stat_(stat), filename_(filename), + p_(p), bkt_(bkt), io_status_(io_status), opts_(opts) {} +}; + +class Filesystem { + public: + File Open(AdapterStat &stat, const std::string &path); + void Open(AdapterStat &stat, File &f, const std::string &path); + size_t Write(File &f, AdapterStat &stat, const void *ptr, + size_t off, size_t total_size, IoStatus &io_status, + IoOptions opts = IoOptions()); + size_t Read(File &f, AdapterStat &stat, void *ptr, + size_t off, size_t total_size, IoStatus &io_status, + IoOptions opts = IoOptions()); + HermesRequest* AWrite(File &f, AdapterStat &stat, const void *ptr, + size_t off, size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts = IoOptions()); + HermesRequest* ARead(File &f, AdapterStat &stat, void *ptr, + size_t off, size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts = IoOptions()); + size_t Wait(uint64_t req_id); + void Wait(std::vector &req_id, std::vector &ret); + off_t Seek(File &f, AdapterStat &stat, SeekMode whence, off_t offset); + off_t Tell(File &f, AdapterStat &stat); + int Sync(File &f, AdapterStat &stat); + int Close(File &f, AdapterStat &stat, bool destroy = true); + + /* + * APIs used internally + * */ + + private: + void _CoordinatedPut(BlobPlacementIter &wi); + void _UncoordinatedPut(BlobPlacementIter &wi); + void _WriteToNewAligned(BlobPlacementIter &write_iter); + void _WriteToNewUnaligned(BlobPlacementIter &write_iter); + void _WriteToExistingAligned(BlobPlacementIter &write_iter); + void _WriteToExistingUnaligned(BlobPlacementIter &write_iter); + void _PutWithFallback(AdapterStat &stat, const std::string &blob_name, + const std::string &filename, u8 *data, size_t size, + size_t offset, IoStatus &io_status_, IoOptions &opts); + size_t _ReadExistingContained(BlobPlacementIter &read_iter); + size_t _ReadExistingPartial(BlobPlacementIter &read_iter); + size_t _ReadNew(BlobPlacementIter &read_iter); + + /* + * The APIs to overload + * */ + public: + virtual void _InitFile(File &f) = 0; + + private: + virtual void _OpenInitStats(File &f, AdapterStat &stat, + bool bucket_exists) = 0; + virtual File _RealOpen(AdapterStat &stat, const std::string &path) = 0; + virtual size_t _RealWrite(const std::string &filename, off_t offset, + size_t size, const u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) = 0; + virtual size_t _RealRead(const std::string &filename, off_t offset, + size_t size, u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) = 0; + virtual void _IoStats(size_t count, IoStatus &io_status, IoOptions &opts) { + (void) count; (void) io_status; (void) opts; + } + virtual int _RealSync(File &f) = 0; + virtual int _RealClose(File &f) = 0; + + /* + * I/O APIs which seek based on the internal AdapterStat st_ptr, + * instead of taking an offset as input. + * */ + + public: + size_t Write(File &f, AdapterStat &stat, const void *ptr, + size_t total_size, IoStatus &io_status, IoOptions opts); + size_t Read(File &f, AdapterStat &stat, void *ptr, + size_t total_size, IoStatus &io_status, IoOptions opts); + HermesRequest* AWrite(File &f, AdapterStat &stat, const void *ptr, + size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts); + HermesRequest* ARead(File &f, AdapterStat &stat, void *ptr, + size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts); + + /* + * Locates the AdapterStat data structure internally, and + * call the underlying APIs which take AdapterStat as input. + * */ + + public: + size_t Write(File &f, bool &stat_exists, const void *ptr, + size_t total_size, IoStatus &io_status, + IoOptions opts = IoOptions()); + size_t Read(File &f, bool &stat_exists, void *ptr, + size_t total_size, IoStatus &io_status, + IoOptions opts = IoOptions()); + size_t Write(File &f, bool &stat_exists, const void *ptr, + size_t off, size_t total_size, IoStatus &io_status, + IoOptions opts = IoOptions()); + size_t Read(File &f, bool &stat_exists, void *ptr, + size_t off, size_t total_size, IoStatus &io_status, + IoOptions opts = IoOptions()); + + HermesRequest* AWrite(File &f, bool &stat_exists, const void *ptr, + size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts); + HermesRequest* ARead(File &f, bool &stat_exists, void *ptr, + size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts); + HermesRequest* AWrite(File &f, bool &stat_exists, const void *ptr, + size_t off, size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts); + HermesRequest* ARead(File &f, bool &stat_exists, void *ptr, + size_t off, size_t total_size, size_t req_id, + IoStatus &io_status, IoOptions opts); + + off_t Seek(File &f, bool &stat_exists, SeekMode whence, off_t offset); + off_t Tell(File &f, bool &stat_exists); + int Sync(File &f, bool &stat_exists); + int Close(File &f, bool &stat_exists, bool destroy = true); +}; + + +} // namespace hermes::adapter::fs + +namespace std { +template <> +struct hash { + std::size_t operator()(const hermes::adapter::fs::File &key) const { + return key.hash(); + } +}; +} // namespace std + +#endif // HERMES_ADAPTER_FILESYSTEM_FILESYSTEM_H_ diff --git a/adapter/posix/metadata_manager.cc b/adapter/filesystem/metadata_manager.cc similarity index 62% rename from adapter/posix/metadata_manager.cc rename to adapter/filesystem/metadata_manager.cc index 3263691ce..9c151ccc0 100644 --- a/adapter/posix/metadata_manager.cc +++ b/adapter/filesystem/metadata_manager.cc @@ -15,55 +15,39 @@ /** * Namespace declarations for cleaner code. */ -using hermes::adapter::posix::AdapterStat; -using hermes::adapter::posix::FileID; -using hermes::adapter::posix::MetadataManager; +using hermes::adapter::fs::AdapterStat; +using hermes::adapter::fs::MetadataManager; -bool MetadataManager::Create(int fh, const AdapterStat &stat) { +bool MetadataManager::Create(const File &f, const AdapterStat &stat) { VLOG(1) << "Create metadata for file handler." << std::endl; - auto ret = metadata.emplace(Convert(fh), stat); + auto ret = metadata.emplace(f, stat); return ret.second; } -bool MetadataManager::Update(int fh, const AdapterStat &stat) { +bool MetadataManager::Update(const File &f, const AdapterStat &stat) { VLOG(1) << "Update metadata for file handler." << std::endl; - auto fileId = Convert(fh); - auto iter = metadata.find(fileId); + auto iter = metadata.find(f); if (iter != metadata.end()) { metadata.erase(iter); - auto ret = metadata.emplace(fileId, stat); + auto ret = metadata.emplace(f, stat); return ret.second; } else { return false; } } -std::pair MetadataManager::Find(int fh) { - auto fileId = Convert(fh); +std::pair MetadataManager::Find(const File &f) { typedef std::pair MetadataReturn; - auto iter = metadata.find(fileId); + auto iter = metadata.find(f); if (iter == metadata.end()) return MetadataReturn(AdapterStat(), false); else return MetadataReturn(iter->second, true); } -FileID MetadataManager::Convert(int fd) { - struct stat st; - MAP_OR_FAIL(__fxstat); - int status = real___fxstat_(_STAT_VER, fd, &st); - if (status == 0) { - return FileID(st.st_dev, st.st_ino); - } else { - // TODO(hari) @error_handling fstat failed invalid fh. - return FileID(); - } -} - -bool MetadataManager::Delete(int fh) { +bool MetadataManager::Delete(const File &f) { VLOG(1) << "Delete metadata for file handler." << std::endl; - auto fileId = Convert(fh); - auto iter = metadata.find(fileId); + auto iter = metadata.find(f); if (iter != metadata.end()) { metadata.erase(iter); return true; diff --git a/adapter/posix/metadata_manager.h b/adapter/filesystem/metadata_manager.h similarity index 87% rename from adapter/posix/metadata_manager.h rename to adapter/filesystem/metadata_manager.h index 68edebd2e..10ec37bf4 100644 --- a/adapter/posix/metadata_manager.h +++ b/adapter/filesystem/metadata_manager.h @@ -22,38 +22,27 @@ #include "constants.h" #include "enumerations.h" #include "interceptor.h" -#include "posix/constants.h" -#include "posix/datastructures.h" +#include "filesystem.h" -namespace hermes::adapter::posix { +namespace hermes::adapter::fs { /** * Metadata manager for POSIX adapter */ class MetadataManager { private: - /** - * Private members - */ - /** - * Maintain a local metadata FileID structure mapped to Adapter Stats. - */ - std::unordered_map metadata; - /** - * hermes attribute to initialize Hermes - */ + std::unordered_map metadata; std::shared_ptr hermes; /** * references of how many times hermes was tried to initialize. */ std::atomic ref; - /** - * MPI attributes - */ + + public: + std::unordered_map request_map; bool is_mpi; int rank; int comm_size; - public: /** * Constructor */ @@ -80,6 +69,14 @@ class MetadataManager { this->is_mpi = is_mpi; char* hermes_config = getenv(kHermesConf); char* hermes_client = getenv(kHermesClient); + char* async_flush_mode = getenv(kHermesAsyncFlush); + + if (async_flush_mode && async_flush_mode[0] == '1') { + global_flushing_mode = FlushingMode::kAsynchronous; + } else { + global_flushing_mode = FlushingMode::kSynchronous; + } + if (this->is_mpi) { MPI_Comm_rank(MPI_COMM_WORLD, &rank); MPI_Comm_size(MPI_COMM_WORLD, &comm_size); @@ -107,11 +104,9 @@ class MetadataManager { MPI_Barrier(MPI_COMM_WORLD); char *stop_daemon = getenv(kStopDaemon); bool shutdown_daemon = true; - if (stop_daemon && stop_daemon[0] == '0') { shutdown_daemon = false; } - hermes->FinalizeClient(shutdown_daemon); } else { hermes->Finalize(true); @@ -119,10 +114,6 @@ class MetadataManager { } ref--; } - /** - * Convert file handler to FileID using the stat. - */ - FileID Convert(int fh); /** * Create a metadata entry for POSIX adapter for a given file handler. @@ -132,7 +123,7 @@ class MetadataManager { * @return true, if operation was successful. * false, if operation was unsuccessful. */ - bool Create(int fh, const AdapterStat& stat); + bool Create(const File &f, const AdapterStat& stat); /** * Update existing metadata entry for POSIX adapter for a given file handler. @@ -141,7 +132,7 @@ class MetadataManager { * @return true, if operation was successful. * false, if operation was unsuccessful or entry doesn't exist. */ - bool Update(int fh, const AdapterStat& stat); + bool Update(const File &f, const AdapterStat& stat); /** * Delete existing metadata entry for POSIX adapter for a given file handler. @@ -149,7 +140,7 @@ class MetadataManager { * @return true, if operation was successful. * false, if operation was unsuccessful. */ - bool Delete(int fh); + bool Delete(const File &f); /** * Find existing metadata entry for POSIX adapter for a given file handler. @@ -157,8 +148,8 @@ class MetadataManager { * @return The metadata entry if exist. * The bool in pair indicated whether metadata entry exists. */ - std::pair Find(int fh); + std::pair Find(const File &f); }; -} // namespace hermes::adapter::posix +} // namespace hermes::adapter::fs #endif // HERMES_ADAPTER_METADATA_MANAGER_H diff --git a/adapter/interceptor.cc b/adapter/interceptor.cc index 29b4e27ab..ca477c476 100644 --- a/adapter/interceptor.cc +++ b/adapter/interceptor.cc @@ -20,8 +20,8 @@ #include "adapter_utils.h" #include "config_parser.h" -namespace fs = std::experimental::filesystem; -const char* kPathExclusions[15] = {"/bin/", "/boot/", "/dev/", "/etc/", +namespace stdfs = std::experimental::filesystem; +const char* kPathExclusions[] = {"/bin/", "/boot/", "/dev/", "/etc/", "/lib/", "/opt/", "/proc/", "/sbin/", "/sys/", "/usr/", "/var/", "/run/", "pipe", "socket:", "anon_inode:"}; @@ -37,9 +37,9 @@ bool populated = false; void PopulateBufferingPath() { char* hermes_config = getenv(kHermesConf); - if (fs::exists(hermes_config)) { + if (stdfs::exists(hermes_config)) { std::string hermes_conf_abs_path = - WeaklyCanonical(fs::path(hermes_config)).string(); + WeaklyCanonical(stdfs::path(hermes_config)).string(); INTERCEPTOR_LIST->hermes_paths_exclusion.push_back(hermes_conf_abs_path); } hermes::Config config_stack = {}; @@ -97,8 +97,8 @@ bool IsTracked(const std::string& path) { } } - for (int i = 0; i < 15; ++i) { - if (abs_path.find(kPathExclusions[i]) == 0) { + for (auto &pth : kPathExclusions) { + if (abs_path.find(pth) != std::string::npos) { return false; } } diff --git a/adapter/interceptor.h b/adapter/interceptor.h index 814622a1b..bb7e12c8b 100644 --- a/adapter/interceptor.h +++ b/adapter/interceptor.h @@ -45,7 +45,7 @@ namespace hermes::adapter { /** * Splits a string given a delimiter */ -inline std::vector StringSplit(char* str, char delimiter) { +inline std::vector StringSplit(const char* str, char delimiter) { std::stringstream ss(str); std::vector v; while (ss.good()) { diff --git a/adapter/mapper/abstract_mapper.h b/adapter/mapper/abstract_mapper.h new file mode 100644 index 000000000..fe29921f1 --- /dev/null +++ b/adapter/mapper/abstract_mapper.h @@ -0,0 +1,83 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +// +// Created by manihariharan on 12/23/20. +// + +#ifndef HERMES_ABSTRACT_ADAPTER_H +#define HERMES_ABSTRACT_ADAPTER_H + +#include "interceptor.h" + +namespace hermes::adapter { + +/** + * Define different types of mappers supported by POSIX Adapter. + * Also define its construction in the MapperFactory. + */ +enum MapperType { + BALANCED = 0 /* Balanced Mapping */ +}; + +struct BlobPlacement { + int rank_; // The rank of the process producing the blob + size_t page_; // The index in the array placements + size_t bucket_off_; // Offset from file start (for FS) + size_t blob_off_; // Offset from blob start + size_t blob_size_; // Size after offset to read + int time_; // The order of the blob in a list of blobs + + std::string CreateBlobName() const { + return std::to_string(page_); + } + + void DecodeBlobName(const std::string &blob_name) { + std::stringstream(blob_name) >> page_; + } + + std::string CreateBlobNameLogEntry(int time) const { + std::stringstream ss; + ss << std::to_string(page_); + ss << "#" << std::to_string(blob_off_); + ss << "#" << std::to_string(blob_size_); + ss << "#" << std::to_string(rank_); + ss << "#" << std::to_string(time); + return ss.str(); + } + + void DecodeBlobNamePerProc(const std::string &blob_name) { + auto str_split = + hermes::adapter::StringSplit(blob_name.data(), '#'); + std::stringstream(str_split[0]) >> page_; + std::stringstream(str_split[1]) >> blob_off_; + std::stringstream(str_split[2]) >> blob_size_; + std::stringstream(str_split[3]) >> rank_; + std::stringstream(str_split[4]) >> rank_; + } +}; + +typedef std::vector BlobPlacements; + +class AbstractMapper { + public: + /** + * This method maps the current Operation to Hermes data structures. + * + * @param file_op, FileStruct, operations for which we are mapping. + * @return a map of FileStruct to Hermes Struct + */ + virtual void map(size_t off, size_t size, BlobPlacements &ps) = 0; +}; +} // namespace hermes::adapter + +#endif // HERMES_ABSTRACT_ADAPTER_H diff --git a/adapter/mpiio/constants.h b/adapter/mapper/balanced_mapper.cc similarity index 53% rename from adapter/mpiio/constants.h rename to adapter/mapper/balanced_mapper.cc index b3c48d633..6784173b6 100644 --- a/adapter/mpiio/constants.h +++ b/adapter/mapper/balanced_mapper.cc @@ -10,28 +10,27 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#ifndef HERMES_MPIIO_COMMON_CONSTANTS_H -#define HERMES_MPIIO_COMMON_CONSTANTS_H +#include "balanced_mapper.h" +#include "constants.h" -#include "mpiio/enumerations.h" +namespace hermes::adapter { -/** - * Constants file for MPIIO adapter. - */ -using hermes::adapter::mpiio::MapperType; +void BalancedMapper::map(size_t off, size_t size, BlobPlacements &ps) { + VLOG(1) << "Mapping File with offset:" << off << " and size:" << size << "." + << std::endl; -/** - * Which mapper to be used by MPIIO adapter. - */ -const MapperType kMapperType = MapperType::BALANCED; + size_t size_mapped = 0; + while (size > size_mapped) { + BlobPlacement p; + p.bucket_off_ = off + size_mapped; + p.page_ = p.bucket_off_ / kPageSize; + p.blob_off_ = p.bucket_off_ % kPageSize; + auto left_size_page = kPageSize - p.blob_off_; + p.blob_size_ = left_size_page < size - size_mapped ? left_size_page + : size - size_mapped; + ps.emplace_back(p); + size_mapped += p.blob_size_; + } +} -/** - * String delimiter - */ -const char kStringDelimiter = '#'; - -/** - * Number of threads for thread pool of async I/O APIs - */ -const int kNumThreads = 1; -#endif // HERMES_MPIIO_COMMON_CONSTANTS_H +} // namespace hermes::adapter diff --git a/adapter/stdio/mapper/balanced_mapper.h b/adapter/mapper/balanced_mapper.h similarity index 89% rename from adapter/stdio/mapper/balanced_mapper.h rename to adapter/mapper/balanced_mapper.h index 7dd4c7507..60e45adbc 100644 --- a/adapter/stdio/mapper/balanced_mapper.h +++ b/adapter/mapper/balanced_mapper.h @@ -14,11 +14,9 @@ #define HERMES_BALANCED_MAPPER_H #include - -#include "stdio/constants.h" #include "abstract_mapper.h" -namespace hermes::adapter::stdio { +namespace hermes::adapter { /** * Implement balanced mapping */ @@ -30,8 +28,8 @@ class BalancedMapper : public AbstractMapper { * @param file_op, FileStruct, operations for which we are mapping. * @return a map of FileStruct to Hermes Struct */ - MapperReturnType map(const FileStruct& file_op) override; + void map(size_t off, size_t size, BlobPlacements &ps) override; }; -} // namespace hermes::adapter::stdio +} // namespace hermes::adapter #endif // HERMES_BALANCED_MAPPER_H diff --git a/adapter/posix/mapper/mapper_factory.h b/adapter/mapper/mapper_factory.h similarity index 90% rename from adapter/posix/mapper/mapper_factory.h rename to adapter/mapper/mapper_factory.h index 9bd07a0ff..de5c1864e 100644 --- a/adapter/posix/mapper/mapper_factory.h +++ b/adapter/mapper/mapper_factory.h @@ -14,12 +14,11 @@ #define HERMES_ADAPTER_FACTORY_H #include "singleton.h" -#include "posix/enumerations.h" - #include "abstract_mapper.h" #include "balanced_mapper.h" +#include "balanced_mapper.cc" -namespace hermes::adapter::posix { +namespace hermes::adapter { class MapperFactory { public: /** @@ -28,7 +27,7 @@ class MapperFactory { * @param type, MapperType, type of mapper to be used by the POSIX adapter. * @return Instance of mapper given a type. */ - std::shared_ptr Get(const MapperType &type) { + AbstractMapper* Get(const MapperType &type) { switch (type) { case MapperType::BALANCED: { return hermes::adapter::Singleton::GetInstance(); @@ -40,5 +39,5 @@ class MapperFactory { return NULL; } }; -} // namespace hermes::adapter::posix +} // namespace hermes::adapter #endif // HERMES_ADAPTER_FACTORY_H diff --git a/adapter/mpiio/CMakeLists.txt b/adapter/mpiio/CMakeLists.txt index f0beb6fbc..556d95567 100644 --- a/adapter/mpiio/CMakeLists.txt +++ b/adapter/mpiio/CMakeLists.txt @@ -1,30 +1,37 @@ project(MPIIOAdapter VERSION ${HERMES_PACKAGE_VERSION}) +include_directories(${HERMES_ADAPTER_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) # MPIIO src code. We only include mpiio.cc as it includes other cc to reduce compilation time. -set(MPIIO_ADAPTER_SRC mpiio.cc) +set(MPIIO_ADAPTER_SRC ${CMAKE_CURRENT_SOURCE_DIR}/mpiio.cc) set(HERMES_MPIIO_ADAPTER_DIR ${HERMES_ADAPTER_DIR}/mpiio) -# Only mpiio.h is the public adapter. -set(MPIIO_ADAPTER_PUBLIC_HEADER ${HERMES_MPIIO_ADAPTER_DIR}/mpiio.h) -# Private headers -set(MPIIO_ADAPTER_PRIVATE_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/metadata_manager.h - ${CMAKE_CURRENT_SOURCE_DIR}/mapper/mapper_factory.h - ${CMAKE_CURRENT_SOURCE_DIR}/mapper/abstract_mapper.h - ${CMAKE_CURRENT_SOURCE_DIR}/mapper/balanced_mapper.h - ${CMAKE_CURRENT_SOURCE_DIR}/datastructures.h - ${CMAKE_CURRENT_SOURCE_DIR}/enumerations.h - ${CMAKE_CURRENT_SOURCE_DIR}/constants.h) +# Public headers +set(MPIIO_ADAPTER_PUBLIC_HEADER + ${HERMES_MPIIO_ADAPTER_DIR}/real_api.h + ${HERMES_MPIIO_ADAPTER_DIR}/fs_api.h) # Add library hermes_mpiio -add_library(hermes_mpiio SHARED ${MPIIO_ADAPTER_PRIVATE_HEADER} ${MPIIO_ADAPTER_PUBLIC_HEADER} ${MPIIO_ADAPTER_SRC}) -target_include_directories(hermes_mpiio PRIVATE ${HERMES_ADAPTER_DIR}) -add_dependencies(hermes_mpiio hermes) -target_link_libraries(hermes_mpiio hermes MPI::MPI_CXX glog::glog stdc++fs) +add_library(hermes_mpiio_backend ${CMAKE_CURRENT_SOURCE_DIR}/fs_api.cc) +add_dependencies(hermes_mpiio_backend hermes) +target_link_libraries(hermes_mpiio_backend hermes MPI::MPI_CXX glog::glog stdc++fs) + +add_library(hermes_mpiio SHARED ${MPIIO_ADAPTER_PUBLIC_HEADER} ${MPIIO_ADAPTER_SRC}) +add_dependencies(hermes_mpiio hermes_mpiio_backend) +target_link_libraries(hermes_mpiio hermes_mpiio_backend) #----------------------------------------------------------------------------- # Add Target(s) to CMake Install #----------------------------------------------------------------------------- +install( + TARGETS + hermes_mpiio_backend + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} +) install( TARGETS hermes_mpiio diff --git a/adapter/mpiio/datastructures.h b/adapter/mpiio/datastructures.h deleted file mode 100644 index f84340d60..000000000 --- a/adapter/mpiio/datastructures.h +++ /dev/null @@ -1,173 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_MPIIO_ADAPTER_DATASTRUCTURES_H -#define HERMES_MPIIO_ADAPTER_DATASTRUCTURES_H - -/** - * Standard header - */ -#include -#include -#include - -/** - * Dependent library header - */ - -/** - * Internal header - */ -#include -#include -#include -#include - -#include - -/** - * Namespace simplification. - */ -namespace hapi = hermes::api; - -namespace hermes::adapter::mpiio { - -/** - * Structure MPIIO adapter uses to define a file state. - */ -struct FileStruct { - /** - * attributes - */ - MPI_File *file_id_; // fileID to identify a file uniquely. - size_t offset_; // file pointer within the file. - size_t size_; // size of data refered in file. - /** - * Constructor - */ - FileStruct() : file_id_(), offset_(0), size_(0) {} /* default constructor */ - FileStruct(MPI_File *file_id, size_t offset, size_t size) - : file_id_(file_id), - offset_(offset), - size_(size) {} /* parameterized constructor */ - FileStruct(const FileStruct &other) - : file_id_(other.file_id_), - offset_(other.offset_), - size_(other.size_) {} /* copy constructor*/ - FileStruct(FileStruct &&other) - : file_id_(other.file_id_), - offset_(other.offset_), - size_(other.size_) {} /* move constructor*/ - /** - * Operators defined - */ - /* Assignment operator. */ - FileStruct &operator=(const FileStruct &other) { - file_id_ = other.file_id_; - offset_ = other.offset_; - size_ = other.size_; - return *this; - } -}; - -/** - * Structure MPIIO adapter uses to define Hermes blob. - */ -struct HermesStruct { - /** - * attributes - */ - std::string blob_name_; - std::string encoded_blob_name_; - size_t offset_; - size_t size_; - /** - * Constructor - */ - HermesStruct() - : blob_name_(), - encoded_blob_name_(), - offset_(0), - size_(0) {} /* default constructor */ - HermesStruct(const HermesStruct &other) - : blob_name_(other.blob_name_), - encoded_blob_name_(other.encoded_blob_name_), - offset_(other.offset_), - size_(other.size_) {} /* copy constructor*/ - HermesStruct(HermesStruct &&other) - : blob_name_(other.blob_name_), - encoded_blob_name_(other.encoded_blob_name_), - offset_(other.offset_), - size_(other.size_) {} /* move constructor*/ - /** - * Operators defined - */ - /* Assignment operator. */ - HermesStruct &operator=(const HermesStruct &other) { - blob_name_ = other.blob_name_; - encoded_blob_name_ = other.encoded_blob_name_; - offset_ = other.offset_; - size_ = other.size_; - return *this; - } -}; - -typedef std::set - StringSet_t; - -/** - * Stat which defines File within MPIIO Adapter. - */ -struct AdapterStat { - /** - * attributes - */ - std::shared_ptr st_bkid; /* bucket associated with the file */ - StringSet_t st_blobs; /* Blobs access in the bucket */ - StringSet_t st_vbuckets; /* vBuckets used in this file */ - i32 ref_count; /* # of time process opens a file */ - int a_mode; /* access mode */ - MPI_Info info; /* Info object (handle) */ - MPI_Comm comm; /* Communicator for the file.*/ - MPI_Offset size; /* total size, in bytes */ - MPI_Offset ptr; /* Current ptr of FILE */ - bool atomicity; /* Consistency semantics for data-access */ - /** - * Constructor - */ - AdapterStat() - : st_bkid(), - st_blobs(CompareBlobs), - ref_count(), - a_mode(), - info(), - comm(), - size(0), - ptr(0), - atomicity(true) {} /* default constructor */ - /** - * Comparator for comparing two blobs. - */ - static bool CompareBlobs(const std::string &a, const std::string &b) { - return std::stol(a) < std::stol(b); - } -}; - - -struct HermesRequest { - std::future return_future; - MPI_Status status; -}; - -} // namespace hermes::adapter::mpiio -#endif // HERMES_MPIIO_ADAPTER_DATASTRUCTURES_H diff --git a/adapter/mpiio/enumerations.h b/adapter/mpiio/enumerations.h deleted file mode 100644 index d202639f8..000000000 --- a/adapter/mpiio/enumerations.h +++ /dev/null @@ -1,25 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_MPIIO_COMMON_ENUMERATIONS_H -#define HERMES_MPIIO_COMMON_ENUMERATIONS_H -/** - * Enumeration for MPIIO adapter. - */ -namespace hermes::adapter::mpiio { -/** - * Define different types of mappers supported by MPIIO Adapter. - * Also define its construction in the MapperFactory. - */ -enum MapperType { BALANCED = 0 /* Balanced Mapping */ }; -} // namespace hermes::adapter::mpiio -#endif // HERMES_MPIIO_COMMON_ENUMERATIONS_H diff --git a/adapter/mpiio/fs_api.cc b/adapter/mpiio/fs_api.cc new file mode 100644 index 000000000..875f603c0 --- /dev/null +++ b/adapter/mpiio/fs_api.cc @@ -0,0 +1,624 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "fs_api.h" + +namespace hermes::adapter::mpiio { + +size_t IoSizeFromCount(int count, MPI_Datatype datatype, IoOptions &opts) { + int datatype_size; + opts.mpi_type_ = datatype; + opts.count_ = count; + MPI_Type_size(datatype, &datatype_size); + return static_cast(count * datatype_size); +} + +int MpiioFS::Read(File &f, AdapterStat &stat, + void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status, IoOptions opts) { + opts.mpi_type_ = datatype; + if (offset + count >= static_cast(stat.st_size)) { + status->count_hi_and_cancelled = 0; + status->count_lo = 0; + return 0; + } + IoStatus io_status; + io_status.mpi_status_ptr_ = status; + size_t total_size = IoSizeFromCount(count, datatype, opts); + Filesystem::Read(f, stat, ptr, offset, + total_size, io_status, opts); + return io_status.mpi_ret_; +} + +int MpiioFS::ARead(File &f, AdapterStat &stat, + void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Request *request, IoOptions opts) { + opts.mpi_type_ = datatype; + IoStatus io_status; + size_t total_size = IoSizeFromCount(count, datatype, opts); + Filesystem::ARead(f, stat, ptr, offset, total_size, + reinterpret_cast(request), + io_status, opts); + return io_status.mpi_ret_; +} + +int MpiioFS::ReadAll(File &f, AdapterStat &stat, + void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status, IoOptions opts) { + opts.mpi_type_ = datatype; + MPI_Barrier(stat.comm); + size_t ret = Read(f, stat, ptr, offset, count, datatype, status, opts); + MPI_Barrier(stat.comm); + return ret; +} + +int MpiioFS::ReadOrdered(File &f, AdapterStat &stat, + void *ptr, int count, + MPI_Datatype datatype, + MPI_Status *status, IoOptions opts) { + opts.mpi_type_ = datatype; + + int total; + MPI_Scan(&count, &total, 1, MPI_INT, MPI_SUM, stat.comm); + MPI_Offset my_offset = total - count; + size_t ret = ReadAll(f, stat, ptr, my_offset, count, datatype, status, opts); + return ret; +} + +int MpiioFS::Write(File &f, AdapterStat &stat, + const void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status, IoOptions opts) { + opts.mpi_type_ = datatype; + IoStatus io_status; + io_status.mpi_status_ptr_ = status; + size_t total_size = IoSizeFromCount(count, datatype, opts); + Filesystem::Write(f, stat, ptr, offset, total_size, + io_status, opts); + return io_status.mpi_ret_; +} + +int MpiioFS::AWrite(File &f, AdapterStat &stat, + const void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Request *request, IoOptions opts) { + opts.mpi_type_ = datatype; + IoStatus io_status; + size_t total_size = IoSizeFromCount(count, datatype, opts); + Filesystem::AWrite(f, stat, ptr, offset, total_size, + reinterpret_cast(request), + io_status, opts); + return io_status.mpi_ret_; +} + +int MpiioFS::WriteAll(File &f, AdapterStat &stat, + const void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status, IoOptions opts) { + opts.mpi_type_ = datatype; + MPI_Barrier(stat.comm); + int ret = Write(f, stat, ptr, offset, count, datatype, status, opts); + MPI_Barrier(stat.comm); + return ret; +} + +int MpiioFS::WriteOrdered(File &f, AdapterStat &stat, + const void *ptr, int count, + MPI_Datatype datatype, + MPI_Status *status, IoOptions opts) { + opts.mpi_type_ = datatype; + int total; + MPI_Scan(&count, &total, 1, MPI_INT, MPI_SUM, stat.comm); + MPI_Offset my_offset = total - count; + size_t ret = WriteAll(f, stat, ptr, my_offset, count, datatype, status, opts); + return ret; +} + +int MpiioFS::AWriteOrdered(File &f, AdapterStat &stat, + const void *ptr, int count, + MPI_Datatype datatype, + MPI_Request *request, IoOptions opts) { + LOG(INFO) << "Starting an asynchronous write" << std::endl; + auto pool = + Singleton::GetInstance(); + HermesRequest *hreq = new HermesRequest(); + auto lambda = + [](MpiioFS *fs, File &f, AdapterStat &stat, + const void *ptr, int count, + MPI_Datatype datatype, MPI_Status *status, + IoOptions opts) { + int ret = fs->WriteOrdered(f, stat, ptr, + count, datatype, status, opts); + return static_cast(ret); + }; + auto func = std::bind(lambda, this, f, stat, ptr, + count, datatype, &hreq->io_status.mpi_status_, + opts); + hreq->return_future = pool->run(func); + auto mdm = Singleton::GetInstance(); + mdm->request_map.emplace(reinterpret_cast(request), hreq); + return MPI_SUCCESS; +} + +int MpiioFS::Wait(MPI_Request *req, MPI_Status *status) { + auto mdm = Singleton::GetInstance(); + auto iter = mdm->request_map.find(reinterpret_cast(req)); + if (iter != mdm->request_map.end()) { + hermes::adapter::fs::HermesRequest *hreq = iter->second; + hreq->return_future.get(); + memcpy(status, + hreq->io_status.mpi_status_ptr_, + sizeof(MPI_Status)); + mdm->request_map.erase(iter); + delete (hreq); + return MPI_SUCCESS; + } + return real_api->MPI_Wait(req, status); +} + +int MpiioFS::WaitAll(int count, MPI_Request *req, MPI_Status *status) { + int ret = 0; + for (int i = 0; i < count; i++) { + auto sub_ret = Wait(&req[i], &status[i]); + if (sub_ret != MPI_SUCCESS) { + ret = sub_ret; + } + } + return ret; +} + +int MpiioFS::Seek(File &f, AdapterStat &stat, + MPI_Offset offset, int whence) { + Filesystem::Seek(f, stat, + MpiioSeekModeConv::Normalize(whence), + offset); + return MPI_SUCCESS; +} + +int MpiioFS::SeekShared(File &f, AdapterStat &stat, + MPI_Offset offset, int whence) { + MPI_Offset sum_offset; + int sum_whence; + int comm_participators; + MPI_Comm_size(stat.comm, &comm_participators); + MPI_Allreduce(&offset, &sum_offset, 1, MPI_LONG_LONG_INT, MPI_SUM, + stat.comm); + MPI_Allreduce(&whence, &sum_whence, 1, MPI_INT, MPI_SUM, + stat.comm); + if (sum_offset / comm_participators != offset) { + LOG(ERROR) + << "Same offset should be passed across the opened file communicator." + << std::endl; + } + if (sum_whence / comm_participators != whence) { + LOG(ERROR) + << "Same whence should be passed across the opened file communicator." + << std::endl; + } + Seek(f, stat, offset, whence); + return 0; +} + +/** + * Variants which internally find the correct offset +* */ + +int MpiioFS::Read(File &f, AdapterStat &stat, + void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status) { + IoOptions opts = IoOptions::DataType(datatype, true); + return Read(f, stat, ptr, Tell(f, stat), count, datatype, status, opts); +} + +int MpiioFS::ARead(File &f, AdapterStat &stat, + void *ptr, int count, MPI_Datatype datatype, MPI_Request *request) { + IoOptions opts = IoOptions::DataType(datatype, true); + return ARead(f, stat, ptr, Tell(f, stat), count, datatype, request, opts); +} + +int MpiioFS::ReadAll(File &f, AdapterStat &stat, + void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status) { + IoOptions opts = IoOptions::DataType(datatype, true); + return ReadAll(f, stat, ptr, Tell(f, stat), count, datatype, status, opts); +} + +int MpiioFS::Write(File &f, AdapterStat &stat, + const void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status) { + IoOptions opts = IoOptions::DataType(datatype, true); + return Write(f, stat, ptr, Tell(f, stat), count, datatype, status, opts); +} + +int MpiioFS::AWrite(File &f, AdapterStat &stat, + const void *ptr, int count, MPI_Datatype datatype, + MPI_Request *request) { + IoOptions opts = IoOptions::DataType(datatype, true); + return AWrite(f, stat, ptr, Tell(f, stat), count, datatype, request, opts); +} + +int MpiioFS::WriteAll(File &f, AdapterStat &stat, + const void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status) { + IoOptions opts = IoOptions::DataType(datatype, true); + return WriteAll(f, stat, ptr, Tell(f, stat), count, datatype, status, opts); +} + + +/** + * Variants which retrieve the stat data structure internally + * */ + +int MpiioFS::Read(File &f, bool &stat_exists, + void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + IoOptions opts = IoOptions::DataType(datatype, false); + return Read(f, stat, ptr, offset, count, datatype, status, opts); +} + +int MpiioFS::ARead(File &f, bool &stat_exists, + void *ptr, size_t offset, + int count, MPI_Datatype datatype, MPI_Request *request) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + IoOptions opts = IoOptions::DataType(datatype, false); + return ARead(f, stat, ptr, offset, count, datatype, request, opts); +} + +int MpiioFS::ReadAll(File &f, bool &stat_exists, + void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + IoOptions opts = IoOptions::DataType(datatype, false); + return ReadAll(f, stat, ptr, offset, count, datatype, status, opts); +} + +int MpiioFS::ReadOrdered(File &f, bool &stat_exists, + void *ptr, int count, + MPI_Datatype datatype, + MPI_Status *status) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + IoOptions opts = IoOptions::DataType(datatype, false); + return ReadOrdered(f, stat, ptr, count, datatype, status, opts); +} + +int MpiioFS::Write(File &f, bool &stat_exists, + const void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + IoOptions opts = IoOptions::DataType(datatype, false); + return Write(f, stat, ptr, offset, count, datatype, status, opts); +} + +int MpiioFS::AWrite(File &f, bool &stat_exists, + const void *ptr, size_t offset, + int count, MPI_Datatype datatype, MPI_Request *request) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + IoOptions opts = IoOptions::DataType(datatype, false); + return AWrite(f, stat, ptr, offset, count, datatype, request, opts); +} + +int MpiioFS::WriteAll(File &f, bool &stat_exists, + const void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + IoOptions opts = IoOptions::DataType(datatype, false); + return WriteAll(f, stat, ptr, offset, count, datatype, status, opts); +} + +int MpiioFS::WriteOrdered(File &f, bool &stat_exists, + const void *ptr, int count, + MPI_Datatype datatype, + MPI_Status *status) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + IoOptions opts = IoOptions::DataType(datatype, false); + return WriteOrdered(f, stat, ptr, count, datatype, status, opts); +} + +int MpiioFS::AWriteOrdered(File &f, bool &stat_exists, + const void *ptr, int count, + MPI_Datatype datatype, + MPI_Request *request) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + IoOptions opts = IoOptions::DataType(datatype, false); + return AWriteOrdered(f, stat, ptr, count, datatype, request, opts); +} + +int MpiioFS::Read(File &f, bool &stat_exists, + void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + return Read(f, stat, ptr, count, datatype, status); +} + +int MpiioFS::ARead(File &f, bool &stat_exists, + void *ptr, int count, MPI_Datatype datatype, + MPI_Request *request) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + return ARead(f, stat, ptr, count, datatype, request); +} + +int MpiioFS::ReadAll(File &f, bool &stat_exists, + void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + return ReadAll(f, stat, ptr, count, datatype, status); +} + +int MpiioFS::Write(File &f, bool &stat_exists, + const void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + return Write(f, stat, ptr, count, datatype, status); +} + +int MpiioFS::AWrite(File &f, bool &stat_exists, + const void *ptr, int count, MPI_Datatype datatype, + MPI_Request *request) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + return AWrite(f, stat, ptr, count, datatype, request); +} + +int MpiioFS::WriteAll(File &f, bool &stat_exists, + const void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + return WriteAll(f, stat, ptr, count, datatype, status); +} + +int MpiioFS::Seek(File &f, bool &stat_exists, + MPI_Offset offset, int whence) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + return Seek(f, stat, offset, whence); +} + +int MpiioFS::SeekShared(File &f, bool &stat_exists, + MPI_Offset offset, int whence) { + auto mdm = Singleton::GetInstance(); + auto [stat, exists] = mdm->Find(f); + if (!exists) { + stat_exists = false; + return -1; + } + stat_exists = true; + return SeekShared(f, stat, offset, whence); +} + +/** + * Internal overrides + * */ + +File MpiioFS::_RealOpen(AdapterStat &stat, const std::string &path) { + File f; + f.mpi_status_ = real_api->MPI_File_open(stat.comm, path.c_str(), stat.amode, + stat.info, &f.mpi_fh_); + if (f.mpi_status_ != MPI_SUCCESS) { + f.status_ = false; + } + // NOTE(llogan): MPI_Info_get does not behave well, so removing + /* + MPI_Info info; + MPI_File_get_info(f.mpi_fh_, &info); + MPI_Info_set(info, "path", path.c_str()); + MPI_File_set_info(f.mpi_fh_, info);*/ + return f; +} + +void MpiioFS::_InitFile(File &f) { + // NOTE(llogan): MPI_Info_get does not behave well, so removing + (void) f; + /*struct stat st; + std::string filename = GetFilenameFromFP(&f.mpi_fh_); + int fd = posix_api->open(filename.c_str(), O_RDONLY); + posix_api->__fxstat(_STAT_VER, fd, &st); + f.st_dev = st.st_dev; + f.st_ino = st.st_ino; + posix_api->close(fd);*/ +} + +void MpiioFS::_OpenInitStats(File &f, AdapterStat &stat, bool bucket_exists) { + (void) bucket_exists; + MPI_Offset size = static_cast(stat.st_size); + MPI_File_get_size(f.mpi_fh_, &size); + stat.st_size = size; + if (stat.amode & MPI_MODE_APPEND) { + stat.st_ptr = stat.st_size; + stat.is_append = true; + } +} + +size_t MpiioFS::_RealWrite(const std::string &filename, off_t offset, + size_t size, const u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) { + LOG(INFO) << "Writing to file: " << filename << " offset: " << offset + << " size:" << size << "." + << " offset:" << offset << "." + << " file_size:" << stdfs::file_size(filename) + << " pid: " << getpid() << std::endl; + MPI_File fh; + int write_count = 0; + io_status.mpi_ret_ = real_api->MPI_File_open(MPI_COMM_SELF, filename.c_str(), + MPI_MODE_RDWR, MPI_INFO_NULL, &fh); + if (io_status.mpi_ret_ != MPI_SUCCESS) { + return 0; + } + + io_status.mpi_ret_ = real_api->MPI_File_seek(fh, offset, MPI_SEEK_SET); + if (io_status.mpi_ret_ != MPI_SUCCESS) { + goto ERROR; + } + io_status.mpi_ret_ = real_api->MPI_File_write(fh, data_ptr, + opts.count_, opts.mpi_type_, + io_status.mpi_status_ptr_); + MPI_Get_count(io_status.mpi_status_ptr_, opts.mpi_type_, &write_count); + if (opts.count_ != write_count) { + LOG(ERROR) << "writing failed: write " << write_count << " of " + << opts.count_ << "." << std::endl; + } + +ERROR: + real_api->MPI_File_close(&fh); + return size; +} + +size_t MpiioFS::_RealRead(const std::string &filename, off_t offset, + size_t size, u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) { + LOG(INFO) << "Read called for filename from destination: " << filename + << " on offset: " << offset << " and size: " << size << "." + << " file_size:" << stdfs::file_size(filename) + << " pid: " << getpid() << std::endl; + MPI_File fh; + int read_count = 0; + io_status.mpi_ret_ = real_api->MPI_File_open(MPI_COMM_SELF, filename.c_str(), + MPI_MODE_RDONLY, MPI_INFO_NULL, &fh); + if (io_status.mpi_ret_ != MPI_SUCCESS) { + return 0; + } + + io_status.mpi_ret_ = real_api->MPI_File_seek(fh, offset, MPI_SEEK_SET); + if (io_status.mpi_ret_ != MPI_SUCCESS) { + goto ERROR; + } + io_status.mpi_ret_ = real_api->MPI_File_read(fh, data_ptr, + opts.count_, opts.mpi_type_, + io_status.mpi_status_ptr_); + MPI_Get_count(io_status.mpi_status_ptr_, + opts.mpi_type_, &read_count); + if (read_count != opts.count_) { + LOG(ERROR) << "reading failed: read " << read_count << " of " << opts.count_ + << "." << std::endl; + } + +ERROR: + real_api->MPI_File_close(&fh); + return size; +} + +void MpiioFS::_IoStats(size_t count, IoStatus &io_status, IoOptions &opts) { + (void) opts; + io_status.mpi_status_ptr_->count_hi_and_cancelled = 0; + io_status.mpi_status_ptr_->count_lo = count; +} + +int MpiioFS::_RealSync(File &f) { + return real_api->MPI_File_sync(f.mpi_fh_); +} + +int MpiioFS::_RealClose(File &f) { + return real_api->MPI_File_close(&f.mpi_fh_); +} + +} // namespace hermes::adapter::mpiio diff --git a/adapter/mpiio/fs_api.h b/adapter/mpiio/fs_api.h new file mode 100644 index 000000000..18c0d2dda --- /dev/null +++ b/adapter/mpiio/fs_api.h @@ -0,0 +1,237 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_ADAPTER_MPIIO_FS_API_H_ +#define HERMES_ADAPTER_MPIIO_FS_API_H_ + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "filesystem/filesystem.h" +#include "filesystem/filesystem.cc" +#include "filesystem/metadata_manager.h" +#include "filesystem/metadata_manager.cc" +#include "real_api.h" +#include "posix/real_api.h" + +namespace hermes::adapter::mpiio { + +using hermes::adapter::fs::AdapterStat; +using hermes::adapter::fs::File; +using hermes::adapter::Singleton; +using hermes::adapter::mpiio::API; +using hermes::adapter::fs::IoOptions; +using hermes::adapter::fs::IoStatus; +using hermes::adapter::fs::HermesRequest; +using hermes::adapter::fs::MetadataManager; +using hermes::adapter::fs::SeekMode; + +class MpiioSeekModeConv { + public: + static SeekMode Normalize(int mpi_seek) { + switch (mpi_seek) { + case MPI_SEEK_SET: return SeekMode::kSet; + case MPI_SEEK_CUR: return SeekMode::kCurrent; + case MPI_SEEK_END: return SeekMode::kEnd; + default: return SeekMode::kNone; + } + } +}; + +class MpiioFS : public hermes::adapter::fs::Filesystem { + private: + API *real_api; + hermes::adapter::posix::API *posix_api; + + public: + MpiioFS() { + real_api = Singleton::GetInstance(); + posix_api = Singleton::GetInstance(); + } + ~MpiioFS() = default; + + void _InitFile(File &f) override; + + + public: + static inline std::string GetFilenameFromFP(MPI_File *fh) { + MPI_Info info; + int status = MPI_File_get_info(*fh, &info); + if (status != MPI_SUCCESS) { + LOG(ERROR) << "MPI_File_get_info on file handler failed." << std::endl; + } + const int kMaxSize = 0xFFF; + int flag; + char filename[kMaxSize] = {0}; + MPI_Info_get(info, "filename", kMaxSize, + filename, &flag); + return filename; + } + + int Read(File &f, AdapterStat &stat, + void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status, IoOptions opts = IoOptions()); + int ARead(File &f, AdapterStat &stat, + void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Request *request, IoOptions opts = IoOptions()); + int ReadAll(File &f, AdapterStat &stat, + void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status, IoOptions opts = IoOptions()); + int ReadOrdered(File &f, AdapterStat &stat, + void *ptr, int count, + MPI_Datatype datatype, + MPI_Status *status, IoOptions opts = IoOptions()); + int Write(File &f, AdapterStat &stat, + const void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status, IoOptions opts = IoOptions()); + int AWrite(File &f, AdapterStat &stat, + const void *ptr, size_t offset, + int count, MPI_Datatype datatype, MPI_Request *request, + IoOptions opts = IoOptions()); + int WriteAll(File &f, AdapterStat &stat, + const void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status, IoOptions opts = IoOptions()); + int WriteOrdered(File &f, AdapterStat &stat, + const void *ptr, int count, + MPI_Datatype datatype, + MPI_Status *status, IoOptions opts = IoOptions()); + int AWriteOrdered(File &f, AdapterStat &stat, + const void *ptr, int count, + MPI_Datatype datatype, + MPI_Request *request, IoOptions opts = IoOptions()); + int Wait(MPI_Request *req, MPI_Status *status); + int WaitAll(int count, MPI_Request *req, MPI_Status *status); + int Seek(File &f, AdapterStat &stat, MPI_Offset offset, int whence); + int SeekShared(File &f, AdapterStat &stat, MPI_Offset offset, int whence); + + /** + * Variants which internally find the correct offset + */ + + public: + int Read(File &f, AdapterStat &stat, + void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status); + int ARead(File &f, AdapterStat &stat, + void *ptr, int count, MPI_Datatype datatype, + MPI_Request *request); + int ReadAll(File &f, AdapterStat &stat, + void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status); + + int Write(File &f, AdapterStat &stat, + const void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status); + int AWrite(File &f, AdapterStat &stat, + const void *ptr, int count, MPI_Datatype datatype, + MPI_Request *request); + int WriteAll(File &f, AdapterStat &stat, + const void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status); + + /** + * Variants which retrieve the stat data structure internally + * */ + + public: + int Read(File &f, bool &stat_exists, + void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status); + int ARead(File &f, bool &stat_exists, + void *ptr, size_t offset, + int count, MPI_Datatype datatype, MPI_Request *request); + int ReadAll(File &f, bool &stat_exists, + void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status); + int ReadOrdered(File &f, bool &stat_exists, + void *ptr, int count, + MPI_Datatype datatype, + MPI_Status *status); + + int Write(File &f, bool &stat_exists, + const void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status); + int AWrite(File &f, bool &stat_exists, + const void *ptr, size_t offset, + int count, MPI_Datatype datatype, MPI_Request *request); + int WriteAll(File &f, bool &stat_exists, + const void *ptr, size_t offset, + int count, MPI_Datatype datatype, + MPI_Status *status); + int WriteOrdered(File &f, bool &stat_exists, + const void *ptr, int count, + MPI_Datatype datatype, + MPI_Status *status); + int AWriteOrdered(File &f, bool &stat_exists, + const void *ptr, int count, + MPI_Datatype datatype, + MPI_Request *request); + + int Read(File &f, bool &stat_exists, + void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status); + int ARead(File &f, bool &stat_exists, + void *ptr, int count, MPI_Datatype datatype, MPI_Request *request); + int ReadAll(File &f, bool &stat_exists, + void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status); + + int Write(File &f, bool &stat_exists, + const void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status); + int AWrite(File &f, bool &stat_exists, + const void *ptr, int count, MPI_Datatype datatype, + MPI_Request *request); + int WriteAll(File &f, bool &stat_exists, + const void *ptr, int count, MPI_Datatype datatype, + MPI_Status *status); + + int Seek(File &f, bool &stat_exists, MPI_Offset offset, int whence); + int SeekShared(File &f, bool &stat_exists, MPI_Offset offset, int whence); + + + /** + * Internal overrides + * */ + + private: + void _OpenInitStats(File &f, AdapterStat &stat, bool bucket_exists) override; + File _RealOpen(AdapterStat &stat, const std::string &path) override; + size_t _RealWrite(const std::string &filename, off_t offset, size_t size, + const u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) override; + size_t _RealRead(const std::string &filename, off_t offset, size_t size, + u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) override; + void _IoStats(size_t count, IoStatus &io_status, IoOptions &opts) override; + int _RealSync(File &f) override; + int _RealClose(File &f) override; +}; + +} // namespace hermes::adapter::mpiio + +#endif // HERMES_ADAPTER_MPIIO_FS_API_H_ diff --git a/adapter/mpiio/mapper/abstract_mapper.h b/adapter/mpiio/mapper/abstract_mapper.h deleted file mode 100644 index 9e79359d6..000000000 --- a/adapter/mpiio/mapper/abstract_mapper.h +++ /dev/null @@ -1,41 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_MPIIO_ADAPTER_ABSTRACT_ADAPTER_H -#define HERMES_MPIIO_ADAPTER_ABSTRACT_ADAPTER_H - -#include "mpiio/datastructures.h" - -/** - * Typedef to simplify the return types - */ -typedef std::vector> - MapperReturnType; - -namespace hermes::adapter::mpiio { -/** - * Interface to define a mapper. - */ -class AbstractMapper { - public: - /** - * This method maps the current Operation to Hermes data structures. - * - * @param file_op, FileStruct, operations for which we are mapping. - * @return a map of FileStruct to Hermes Struct - */ - virtual MapperReturnType map(const FileStruct& file_op) = 0; -}; -} // namespace hermes::adapter::mpiio - -#endif // HERMES_MPIIO_ADAPTER_ABSTRACT_ADAPTER_H diff --git a/adapter/mpiio/mapper/balanced_mapper.cc b/adapter/mpiio/mapper/balanced_mapper.cc deleted file mode 100644 index 32ce95d7f..000000000 --- a/adapter/mpiio/mapper/balanced_mapper.cc +++ /dev/null @@ -1,46 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "balanced_mapper.h" - -/** - * Namespace declaration for cleaner code. - */ -using hermes::adapter::mpiio::BalancedMapper; -using hermes::adapter::mpiio::FileStruct; -using hermes::adapter::mpiio::HermesStruct; - -MapperReturnType BalancedMapper::map(const FileStruct& file_op) { - VLOG(1) << "Mapping File with offset:" << file_op.offset_ - << " and size:" << file_op.size_ << "." << std::endl; - - auto mapper_return = MapperReturnType(); - size_t size_mapped = 0; - while (file_op.size_ > size_mapped) { - FileStruct file; - file.file_id_ = file_op.file_id_; - HermesStruct hermes; - file.offset_ = file_op.offset_ + size_mapped; - size_t page_index = file.offset_ / kPageSize; - hermes.offset_ = file.offset_ % kPageSize; - auto left_size_page = kPageSize - hermes.offset_; - hermes.size_ = left_size_page < file_op.size_ - size_mapped - ? left_size_page - : file_op.size_ - size_mapped; - - file.size_ = hermes.size_; - hermes.blob_name_ = std::to_string(page_index + 1); - mapper_return.emplace_back(file, hermes); - size_mapped += hermes.size_; - } - return mapper_return; -} diff --git a/adapter/mpiio/mapper/balanced_mapper.h b/adapter/mpiio/mapper/balanced_mapper.h deleted file mode 100644 index d86f32a7c..000000000 --- a/adapter/mpiio/mapper/balanced_mapper.h +++ /dev/null @@ -1,37 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_MPIIO_ADAPTER_BALANCED_MAPPER_H -#define HERMES_MPIIO_ADAPTER_BALANCED_MAPPER_H - -#include - -#include "mpiio/constants.h" -#include "abstract_mapper.h" - -namespace hermes::adapter::mpiio { -/** - * Implement balanced mapping - */ -class BalancedMapper : public AbstractMapper { - public: - /** - * This method maps the current Operation to Hermes data structures. - * - * @param file_op, FileStruct, operations for which we are mapping. - * @return a map of FileStruct to Hermes Struct - */ - MapperReturnType map(const FileStruct& file_op) override; -}; -} // namespace hermes::adapter::mpiio - -#endif // HERMES_MPIIO_ADAPTER_BALANCED_MAPPER_H diff --git a/adapter/mpiio/mapper/mapper_factory.h b/adapter/mpiio/mapper/mapper_factory.h deleted file mode 100644 index 7e602de0b..000000000 --- a/adapter/mpiio/mapper/mapper_factory.h +++ /dev/null @@ -1,44 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_MPIIO_ADAPTER_ADAPTER_FACTORY_H -#define HERMES_MPIIO_ADAPTER_ADAPTER_FACTORY_H - -#include "singleton.h" -#include "mpiio/enumerations.h" - -#include "abstract_mapper.h" -#include "balanced_mapper.h" - -namespace hermes::adapter::mpiio { -class MapperFactory { - public: - /** - * Return the instance of mapper given a type. Uses factory pattern. - * - * @param type, MapperType, type of mapper to be used by the MPIIO adapter. - * @return Instance of mapper given a type. - */ - std::shared_ptr Get(const MapperType &type) { - switch (type) { - case MapperType::BALANCED: { - return hermes::adapter::Singleton::GetInstance(); - } - default: { - // TODO(hari): @errorhandling Mapper not implemented - } - } - return NULL; - } -}; -} // namespace hermes::adapter::mpiio -#endif // HERMES_MPIIO_ADAPTER_ADAPTER_FACTORY_H diff --git a/adapter/mpiio/metadata_manager.cc b/adapter/mpiio/metadata_manager.cc deleted file mode 100644 index 3f6e215e0..000000000 --- a/adapter/mpiio/metadata_manager.cc +++ /dev/null @@ -1,86 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "metadata_manager.h" - - - -/** - * Namespace declarations for cleaner code. - */ -using hermes::adapter::mpiio::AdapterStat; -using hermes::adapter::mpiio::HermesStruct; -using hermes::adapter::mpiio::MetadataManager; - -bool MetadataManager::Create(MPI_File *fh, const AdapterStat &stat) { - VLOG(1) << "Create metadata for file handler." << std::endl; - auto ret = metadata.emplace(*fh, stat); - return ret.second; -} - -bool MetadataManager::Update(MPI_File *fh, const AdapterStat &stat) { - VLOG(1) << "Update metadata for file handler." << std::endl; - auto iter = metadata.find(*fh); - if (iter != metadata.end()) { - metadata.erase(iter); - auto ret = metadata.emplace(*fh, stat); - return ret.second; - } else { - return false; - } -} - -std::pair MetadataManager::Find(MPI_File *fh) { - typedef std::pair MetadataReturn; - auto iter = metadata.find(*fh); - if (iter == metadata.end()) - return MetadataReturn(AdapterStat(), false); - else - return MetadataReturn(iter->second, true); -} - -bool MetadataManager::Delete(MPI_File *fh) { - VLOG(1) << "Delete metadata for file handler." << std::endl; - auto iter = metadata.find(*fh); - if (iter != metadata.end()) { - metadata.erase(iter); - return true; - } else { - return false; - } -} - -std::string MetadataManager::EncodeBlobNameLocal(HermesStruct hermes_struct) { - LOG(INFO) << "Encode Blob:" << hermes_struct.blob_name_ - << " for hermes blobs." << std::endl; - return hermes_struct.blob_name_ + kStringDelimiter + - std::to_string(hermes_struct.offset_) + kStringDelimiter + - std::to_string(hermes_struct.size_) + kStringDelimiter + - std::to_string(rank); -} - -std::pair MetadataManager::DecodeBlobNameLocal( - std::string &encoded_blob_name) { - HermesStruct hermes_struct; - auto str_split = - hermes::adapter::StringSplit(encoded_blob_name.data(), kStringDelimiter); - hermes_struct.encoded_blob_name_ = encoded_blob_name; - hermes_struct.blob_name_ = encoded_blob_name; - int blob_rank = -1; - if (str_split.size() == 4) { - hermes_struct.blob_name_ = str_split[0]; - hermes_struct.offset_ = std::stoi(str_split[1]); - hermes_struct.size_ = std::stoi(str_split[2]); - blob_rank = std::stoi(str_split[3]); - } - return std::pair(blob_rank, hermes_struct); -} diff --git a/adapter/mpiio/metadata_manager.h b/adapter/mpiio/metadata_manager.h deleted file mode 100644 index 3b7d2af63..000000000 --- a/adapter/mpiio/metadata_manager.h +++ /dev/null @@ -1,153 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_MPIIO_ADAPTER_METADATA_MANAGER_H -#define HERMES_MPIIO_ADAPTER_METADATA_MANAGER_H - -#include -#include - -#include - -#include "constants.h" -#include "enumerations.h" -#include "interceptor.h" -#include "mpiio/constants.h" -#include "mpiio/datastructures.h" - -namespace hermes::adapter::mpiio { -/** - * Metadata manager for MPIIO adapter - */ -class MetadataManager { - private: - /** - * Private members - */ - /** - * Maintain a local metadata FileID structure mapped to Adapter Stats. - */ - std::unordered_map metadata; - /** - * hermes attribute to initialize Hermes - */ - std::shared_ptr hermes; - /** - * references of how many times hermes was tried to initialize. - */ - std::atomic ref; - - public: - /** - * MPI attributes - */ - int rank; - int comm_size; - /** - * Maintain a local metadata MPI_Request and HermesRequest. - */ - std::unordered_map request_map; - /** - * Constructor - */ - MetadataManager() - : metadata(), ref(0), rank(0), comm_size(1), request_map() {} - /** - * Get the instance of hermes. - */ - std::shared_ptr& GetHermes() { return hermes; } - - /** - * Initialize hermes. Get the kHermesConf from environment else get_env - * returns NULL which is handled internally by hermes. Initialize hermes in - * daemon mode. Keep a reference of how many times Initialize is called. - * Within the adapter, Initialize is called from fopen. - */ - void InitializeHermes() { - if (ref == 0) { - char* hermes_config = getenv(kHermesConf); - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &comm_size); - // TODO(chogan): Need a better way to distinguish between client and - // daemon. https://github.com/HDFGroup/hermes/issues/206 - if (comm_size > 1) { - hermes = hermes::InitHermesClient(hermes_config); - } else { - hermes = hermes::InitHermesDaemon(hermes_config); - } - INTERCEPTOR_LIST->SetupAdapterMode(); - } - ref++; - } - /** - * Finalize hermes and close rpc if reference is equal to one. Else just - * decrement the ref counter. - */ - void FinalizeHermes() { - if (ref == 1) { - hermes->FinalizeClient(); - } - ref--; - } - - /** - * Create a metadata entry for MPIIO adapter for a given file handler. - * @param fh, FILE*, original file handler of the file on the destination - * filesystem. - * @param stat, AdapterStat, MPIIO Adapter version of Stat data structure. - * @return true, if operation was successful. - * false, if operation was unsuccessful. - */ - bool Create(MPI_File* fh, const AdapterStat& stat); - - /** - * Update existing metadata entry for MPIIO adapter for a given file handler. - * @param fh, FILE*, original file handler of the file on the destination. - * @param stat, AdapterStat, MPIIO Adapter version of Stat data structure. - * @return true, if operation was successful. - * false, if operation was unsuccessful or entry doesn't exist. - */ - bool Update(MPI_File* fh, const AdapterStat& stat); - - /** - * Delete existing metadata entry for MPIIO adapter for a given file handler. - * @param fh, FILE*, original file handler of the file on the destination. - * @return true, if operation was successful. - * false, if operation was unsuccessful. - */ - bool Delete(MPI_File* fh); - - /** - * Find existing metadata entry for MPIIO adapter for a given file handler. - * @param fh, FILE*, original file handler of the file on the destination. - * @return The metadata entry if exist. - * The bool in pair indicated whether metadata entry exists. - */ - std::pair Find(MPI_File* fh); - /** - * Encode a given Hermes Struct from Mapping engine to Local Blob Name. - * @param hermes_struct, HermesStruct, structure containing hermes - * representatrion of file. - * @return string encoded with process local information. - */ - std::string EncodeBlobNameLocal(HermesStruct hermes_struct); - /** - * Decodes a encoded blob name string into hermesstruct - * @param encoded_blob_name, std::string, encoded blob name string. - * @return hermes struct with blob_name, rank, offset, and size - */ - std::pair DecodeBlobNameLocal( - std::string& encoded_blob_name); -}; -} // namespace hermes::adapter::mpiio - -#endif // HERMES_MPIIO_ADAPTER_METADATA_MANAGER_H diff --git a/adapter/mpiio/mpiio.cc b/adapter/mpiio/mpiio.cc index 7419b3190..afed8452e 100644 --- a/adapter/mpiio/mpiio.cc +++ b/adapter/mpiio/mpiio.cc @@ -10,1120 +10,405 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "mpiio/mpiio.h" +bool mpiio_intercepted = true; + +#include +#include +#include + +#include "real_api.h" +#include "fs_api.h" + +#include "constants.h" +#include "singleton.h" +#include "interceptor.h" +#include "interceptor.cc" #include "thread_pool.h" -#include "adapter_utils.cc" -#include "mpiio/mapper/balanced_mapper.cc" /** * Namespace declarations */ using hermes::ThreadPool; -using hermes::adapter::mpiio::AdapterStat; -using hermes::adapter::mpiio::FileStruct; -using hermes::adapter::mpiio::HermesRequest; -using hermes::adapter::mpiio::MapperFactory; -using hermes::adapter::mpiio::MetadataManager; +using hermes::adapter::fs::MetadataManager; +using hermes::adapter::fs::File; +using hermes::adapter::mpiio::API; +using hermes::adapter::mpiio::MpiioFS; +using hermes::adapter::mpiio::MpiioSeekModeConv; +using hermes::adapter::Singleton; namespace hapi = hermes::api; -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; using hermes::adapter::WeaklyCanonical; /** * Internal Functions. */ -inline std::string GetFilenameFromFP(MPI_File *fh) { - MPI_Info info_out; - int status = MPI_File_get_info(*fh, &info_out); - if (status != MPI_SUCCESS) { - LOG(ERROR) << "MPI_File_get_info on file handler failed." << std::endl; - } - const int kMaxSize = 0xFFF; - int flag; - char filename[kMaxSize]; - MPI_Info_get(info_out, "filename", kMaxSize, filename, &flag); - return filename; -} + inline bool IsTracked(MPI_File *fh) { if (hermes::adapter::exit) return false; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fh); - return existing.second; -} - -int simple_open(MPI_Comm &comm, const char *user_path, int &amode, - MPI_Info &info, MPI_File *fh) { - std::string path_str = WeaklyCanonical(user_path).string(); - - LOG(INFO) << "Open file for filename " << path_str << " in mode " << amode - << std::endl; - int ret = MPI_SUCCESS; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fh); - if (!existing.second) { - LOG(INFO) << "File not opened before by adapter" << std::endl; - AdapterStat stat; - stat.ref_count = 1; - stat.a_mode = amode; - MPI_File_get_size(*fh, &stat.size); - if (amode & MPI_MODE_APPEND) { - stat.ptr = stat.size; - } - stat.info = info; - stat.comm = comm; - hapi::Context ctx; - stat.st_bkid = std::make_shared(path_str, mdm->GetHermes(), - ctx); - mdm->Create(fh, stat); - } else { - LOG(INFO) << "File opened before by adapter" << std::endl; - existing.first.ref_count++; - mdm->Update(fh, existing.first); - } - return ret; + auto mdm = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); + File f; f.mpi_fh_ = (*fh); fs_api->_InitFile(f); + auto [stat, exists] = mdm->Find(f); + return exists; } -int open_internal(MPI_Comm &comm, const char *path, int &amode, MPI_Info &info, - MPI_File *fh) { - int ret; - MAP_OR_FAIL(MPI_File_open); - ret = real_MPI_File_open_(comm, path, amode, info, fh); - if (ret == MPI_SUCCESS) { - ret = simple_open(comm, path, amode, info, fh); - } - return ret; -} -size_t perform_file_read(const char *filename, size_t file_offset, void *ptr, - size_t ptr_offset, int count, MPI_Datatype datatype) { - LOG(INFO) << "Read called for filename from destination: " << filename - << " on offset: " << file_offset << " and count: " << count - << std::endl; - INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); - MPI_File fh; - int status = MPI_File_open(MPI_COMM_SELF, filename, MPI_MODE_RDONLY, - MPI_INFO_NULL, &fh); - int read_size = 0; - if (status == MPI_SUCCESS) { - status = MPI_File_seek(fh, file_offset, MPI_SEEK_SET); - if (status == MPI_SUCCESS) { - MPI_Status read_status; - status = MPI_File_read(fh, (char *)ptr + ptr_offset, count, datatype, - &read_status); - MPI_Get_count(&read_status, datatype, &read_size); - if (read_size != count) { - LOG(ERROR) << "reading failed: read " << read_size << " of " << count - << "." << std::endl; - } - } - status = MPI_File_close(&fh); - } - INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename); - return read_size; -} - -size_t perform_file_write(std::string &filename, int offset, int count, - MPI_Datatype datatype, unsigned char *data_ptr) { - LOG(INFO) << "Writing to file: " << filename << " offset: " << offset - << " of count:" << count << " datatype:" << datatype << "." - << std::endl; - INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); - MPI_File fh; - int status = MPI_File_open(MPI_COMM_SELF, filename.c_str(), MPI_MODE_RDWR, - MPI_INFO_NULL, &fh); - int write_size = 0; - if (fh != nullptr) { - status = MPI_File_seek(fh, offset, MPI_SEEK_SET); - if (status == 0) { - MPI_Status write_status; - status = MPI_File_write(fh, data_ptr, count, datatype, &write_status); - MPI_Get_count(&write_status, datatype, &write_size); - if (write_size != count) { - LOG(ERROR) << "writing failed: wrote " << write_size << " of " << count - << "." << std::endl; - } - status = MPI_File_close(&fh); - } - } - INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename); - return write_size; -} - -std::pair write_internal(std::pair &existing, - const void *ptr, int count, - MPI_Datatype datatype, MPI_File *fp, - MPI_Status *mpi_status, - bool is_collective = false) { - (void)is_collective; - LOG(INFO) << "Write called for filename: " - << existing.first.st_bkid->GetName() - << " on offset: " << existing.first.ptr << " and count: " << count - << std::endl; - size_t ret; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto mapper = MapperFactory().Get(kMapperType); - int datatype_size; - MPI_Type_size(datatype, &datatype_size); - size_t total_size = datatype_size * count; - auto mapping = mapper->map(FileStruct(fp, existing.first.ptr, total_size)); - size_t data_offset = 0; - auto filename = existing.first.st_bkid->GetName(); - LOG(INFO) << "Mapping for write has " << mapping.size() << " mapping." - << std::endl; - for (const auto &item : mapping) { - hapi::Context ctx; - auto index = std::stol(item.second.blob_name_) - 1; - unsigned char *put_data_ptr = (unsigned char *)ptr + data_offset; - size_t put_data_ptr_size = item.first.size_; - - if (item.second.size_ == kPageSize) { - LOG(INFO) << "Create or Overwrite blob " << item.second.blob_name_ - << " of size:" << item.second.size_ << "." << std::endl; - auto status = existing.first.st_bkid->Put( - item.second.blob_name_, put_data_ptr, put_data_ptr_size, ctx); - if (status.Failed()) { - perform_file_write(filename, item.first.offset_, put_data_ptr_size, - MPI_CHAR, put_data_ptr); - } else { - existing.first.st_blobs.emplace(item.second.blob_name_); - } - } else { - LOG(INFO) << "Writing blob " << item.second.blob_name_ - << " of size:" << item.second.size_ << "." << std::endl; - auto blob_exists = - existing.first.st_bkid->ContainsBlob(item.second.blob_name_); - if (blob_exists) { - hapi::Blob temp(0); - auto existing_blob_size = - existing.first.st_bkid->Get(item.second.blob_name_, temp, ctx); - LOG(INFO) << "blob " << item.second.blob_name_ - << " of size:" << existing_blob_size << " exists." - << std::endl; - hapi::Blob existing_data(existing_blob_size); - existing.first.st_bkid->Get(item.second.blob_name_, existing_data, ctx); - memcpy(existing_data.data() + item.second.offset_, put_data_ptr, - put_data_ptr_size); - auto status = existing.first.st_bkid->Put(item.second.blob_name_, - existing_data, ctx); - if (status.Failed()) { - LOG(INFO) << "blob " << item.second.blob_name_ - << " put failed for size" << existing_blob_size - << ". Writing to file system directly." << std::endl; - perform_file_write(filename, index * kPageSize, existing_blob_size, - MPI_CHAR, existing_data.data()); - } else { - existing.first.st_blobs.emplace(item.second.blob_name_); - } - } else { - LOG(INFO) << "blob " << item.second.blob_name_ - << " does not exists and we are writing partial blob." - << std::endl; - std::string process_local_blob_name = - mdm->EncodeBlobNameLocal(item.second); - auto vbucket_name = filename + "#" + item.second.blob_name_; - auto vbucket = hapi::VBucket(vbucket_name, mdm->GetHermes()); - existing.first.st_vbuckets.emplace(vbucket_name); - auto blob_names = vbucket.GetLinks(ctx); - LOG(INFO) << "vbucket with blobname " << item.second.blob_name_ - << " does not exists." << std::endl; - auto status = existing.first.st_bkid->Put( - process_local_blob_name, put_data_ptr, put_data_ptr_size, ctx); - if (status.Failed()) { - LOG(INFO) << "blob " << process_local_blob_name - << " put failed for size" << put_data_ptr_size - << ". Writing to file system directly." << std::endl; - perform_file_write(filename, item.first.offset_, put_data_ptr_size, - MPI_CHAR, put_data_ptr); - } else { - existing.first.st_blobs.emplace(process_local_blob_name); - vbucket.Link(process_local_blob_name, - existing.first.st_bkid->GetName()); - } - LOG(INFO) << "clean existing blob list metadata." << std::endl; - if (!blob_names.empty()) { - LOG(INFO) << "vbucket with blobname " << item.second.blob_name_ - << " exists." << std::endl; - for (auto &blob_name : blob_names) { - auto hermes_struct = mdm->DecodeBlobNameLocal(blob_name); - if (((hermes_struct.second.offset_ < item.second.offset_) && - (hermes_struct.second.offset_ + hermes_struct.second.size_ > - item.second.offset_))) { - // partially contained second half - hapi::Blob existing_data(item.second.offset_ - - hermes_struct.second.offset_); - existing.first.st_bkid->Get(item.second.blob_name_, existing_data, - ctx); - status = existing.first.st_bkid->Put(item.second.blob_name_, - existing_data, ctx); - if (status.Failed()) { - LOG(ERROR) << "Put Failed on adapter." << std::endl; - } - } else if (item.second.offset_ < hermes_struct.second.offset_ && - item.second.offset_ + item.second.size_ > - hermes_struct.second.offset_) { - // partially contained first half - hapi::Blob existing_data(hermes_struct.second.size_); - existing.first.st_bkid->Get(item.second.blob_name_, existing_data, - ctx); - existing_data.erase( - existing_data.begin(), - existing_data.begin() + - (hermes_struct.second.offset_ - item.second.offset_)); - status = existing.first.st_bkid->Put(item.second.blob_name_, - existing_data, ctx); - if (status.Failed()) { - LOG(ERROR) << "Put Failed on adapter." << std::endl; - } - } else if (hermes_struct.second.offset_ > item.second.offset_ && - hermes_struct.second.offset_ + - hermes_struct.second.size_ < - item.second.offset_ + item.second.size_) { - // fully contained - status = - existing.first.st_bkid->DeleteBlob(item.second.blob_name_); - if (status.Failed()) { - LOG(ERROR) << "Delete blob Failed on adapter." << std::endl; - } - existing.first.st_blobs.erase(item.second.blob_name_); - } else { - // no overlap - } - } - } - vbucket.Release(); - } - } - data_offset += item.first.size_; - } - existing.first.ptr += data_offset; - existing.first.size = existing.first.size >= existing.first.ptr - ? existing.first.size - : existing.first.ptr; - mdm->Update(fp, existing.first); - ret = data_offset; - mpi_status->count_hi_and_cancelled = 0; - mpi_status->count_lo = ret; - return std::pair(MPI_SUCCESS, ret); -} - -std::pair read_internal(std::pair &existing, - void *ptr, int count, - MPI_Datatype datatype, MPI_File *fp, - MPI_Status *mpi_status, - bool is_collective = false) { - (void)is_collective; - LOG(INFO) << "Read called for filename: " << existing.first.st_bkid->GetName() - << " on offset: " << existing.first.ptr << " and size: " << count - << std::endl; - if (existing.first.ptr >= existing.first.size) { - mpi_status->count_hi_and_cancelled = 0; - mpi_status->count_lo = 0; - return std::pair(MPI_SUCCESS, 0); - } - int datatype_size; - MPI_Type_size(datatype, &datatype_size); - size_t total_size = datatype_size * count; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto mapper = MapperFactory().Get(kMapperType); - auto mapping = mapper->map(FileStruct(fp, existing.first.ptr, total_size)); - size_t total_read_size = 0; - auto filename = existing.first.st_bkid->GetName(); - LOG(INFO) << "Mapping for read has " << mapping.size() << " mapping." - << std::endl; - for (const auto &item : mapping) { - hapi::Context ctx; - auto blob_exists = - existing.first.st_bkid->ContainsBlob(item.second.blob_name_); - hapi::Blob read_data(0); - size_t read_size = 0; - if (blob_exists) { - LOG(INFO) << "Blob exists and need to read from Hermes from blob: " - << item.second.blob_name_ << "." << std::endl; - auto exiting_blob_size = - existing.first.st_bkid->Get(item.second.blob_name_, read_data, ctx); - - read_data.resize(exiting_blob_size); - existing.first.st_bkid->Get(item.second.blob_name_, read_data, ctx); - bool contains_blob = exiting_blob_size > item.second.offset_; - if (contains_blob) { - read_size = read_data.size() < item.second.offset_ + item.second.size_ - ? exiting_blob_size - item.second.offset_ - : item.second.size_; - LOG(INFO) << "Blob have data and need to read from hemes " - "blob: " - << item.second.blob_name_ << " offset:" << item.second.offset_ - << " size:" << read_size << "." << std::endl; - memcpy((char *)ptr + total_read_size, - read_data.data() + item.second.offset_, read_size); - if (read_size < item.second.size_) { - contains_blob = true; - } else { - contains_blob = false; - } - } else { - LOG(INFO) << "Blob does not have data and need to read from original " - "filename: " - << filename << " offset:" << item.first.offset_ - << " size:" << item.first.size_ << "." << std::endl; - auto file_read_size = perform_file_read( - filename.c_str(), item.first.offset_, ptr, total_read_size, - (int)item.second.size_, MPI_CHAR); - read_size += file_read_size; - } - if (contains_blob && fs::exists(filename) && - fs::file_size(filename) >= item.first.offset_ + item.first.size_) { - LOG(INFO) << "Blob does not have data and need to read from original " - "filename: " - << filename << " offset:" << item.first.offset_ + read_size - << " size:" << item.second.size_ - read_size << "." - << std::endl; - auto new_read_size = - perform_file_read(filename.c_str(), item.first.offset_, ptr, - total_read_size + read_size, - item.second.size_ - read_size, MPI_CHAR); - read_size += new_read_size; - } - } else { - auto vbucket_name = filename + "#" + item.second.blob_name_; - hapi::VBucket vbucket(vbucket_name, mdm->GetHermes()); - auto blob_names = vbucket.GetLinks(ctx); - if (!blob_names.empty()) { - LOG(INFO) << "vbucket with blobname " << item.second.blob_name_ - << " exists." << std::endl; - for (auto &blob_name : blob_names) { - auto hermes_struct = mdm->DecodeBlobNameLocal(blob_name); - if (((hermes_struct.second.offset_ <= item.second.offset_) && - (hermes_struct.second.offset_ + hermes_struct.second.size_ >= - item.second.offset_) && - (hermes_struct.second.size_ - item.second.offset_ - - hermes_struct.second.offset_ > - 0))) { - // partially contained second half - hapi::Blob existing_data(hermes_struct.second.size_); - existing.first.st_bkid->Get(hermes_struct.second.encoded_blob_name_, - existing_data, ctx); - auto offset_to_cp = - item.second.offset_ - hermes_struct.second.offset_; - memcpy((char *)ptr + (item.first.offset_ - existing.first.ptr), - existing_data.data() + offset_to_cp, - hermes_struct.second.size_ - offset_to_cp); - read_size += hermes_struct.second.size_ - offset_to_cp; - } else if (item.second.offset_ < hermes_struct.second.offset_ && - item.second.offset_ + item.second.size_ > - hermes_struct.second.offset_) { - // partially contained first half - hapi::Blob existing_data(hermes_struct.second.size_); - existing.first.st_bkid->Get(hermes_struct.second.encoded_blob_name_, - existing_data, ctx); - memcpy((char *)ptr + (item.first.offset_ - existing.first.ptr), - existing_data.data(), - (hermes_struct.second.offset_ - item.second.offset_)); - read_size += hermes_struct.second.offset_ - item.second.offset_; - } else if (hermes_struct.second.offset_ > item.second.offset_ && - hermes_struct.second.offset_ + hermes_struct.second.size_ < - item.second.offset_ + item.second.size_) { - // fully contained - hapi::Blob existing_data(hermes_struct.second.size_); - existing.first.st_bkid->Get(hermes_struct.second.encoded_blob_name_, - existing_data, ctx); - memcpy((char *)ptr + (item.first.offset_ - existing.first.ptr), - existing_data.data(), hermes_struct.second.size_); - read_size += hermes_struct.second.size_; - } else { - // no overlap - } - if (read_size == item.second.size_) break; - } - } else if (fs::exists(filename) && - fs::file_size(filename) >= - item.first.offset_ + item.first.size_) { - LOG(INFO) << "Blob does not exists and need to read from original " - "filename: " - << filename << " offset:" << item.first.offset_ - << " size:" << item.first.size_ << "." << std::endl; - read_size = - perform_file_read(filename.c_str(), item.first.offset_, ptr, - total_read_size, item.first.size_, MPI_CHAR); - } - vbucket.Release(); - } - if (read_size > 0) { - total_read_size += read_size; - } - } - existing.first.ptr += total_read_size; - mdm->Update(fp, existing.first); - mpi_status->count_hi_and_cancelled = 0; - mpi_status->count_lo = total_read_size; - return std::pair(MPI_SUCCESS, total_read_size); -} /** * MPI */ int HERMES_DECL(MPI_Init)(int *argc, char ***argv) { - MAP_OR_FAIL(MPI_Init); - int status = real_MPI_Init_(argc, argv); + auto real_api = Singleton::GetInstance(); + int status = real_api->MPI_Init(argc, argv); if (status == 0) { LOG(INFO) << "MPI Init intercepted." << std::endl; - auto mdm = hermes::adapter::Singleton::GetInstance(); - mdm->InitializeHermes(); - auto pool = - hermes::adapter::Singleton::GetInstance(kNumThreads); + auto mdm = Singleton::GetInstance(); + mdm->InitializeHermes(true); + Singleton::GetInstance(hermes::adapter::fs::kNumThreads); } return status; } int HERMES_DECL(MPI_Finalize)(void) { LOG(INFO) << "MPI Finalize intercepted." << std::endl; - auto mdm = hermes::adapter::Singleton::GetInstance(); + auto real_api = Singleton::GetInstance(); + auto mdm = Singleton::GetInstance(); mdm->FinalizeHermes(); - MAP_OR_FAIL(MPI_Finalize); - int status = real_MPI_Finalize_(); + int status = real_api->MPI_Finalize(); return status; } int HERMES_DECL(MPI_Wait)(MPI_Request *req, MPI_Status *status) { - int ret; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto iter = mdm->request_map.find(req); - if (iter != mdm->request_map.end()) { - ret = iter->second->return_future.get(); - memcpy(status, &iter->second->status, sizeof(MPI_Status)); - auto h_req = iter->second; - mdm->request_map.erase(iter); - delete (h_req); - } else { - MAP_OR_FAIL(MPI_Wait); - ret = real_MPI_Wait_(req, status); - } - return ret; + auto fs_api = Singleton::GetInstance(); + return fs_api->Wait(req, status); } int HERMES_DECL(MPI_Waitall)(int count, MPI_Request *req, MPI_Status *status) { - int ret = 0; - for (int i = 0; i < count; i++) { - auto sub_ret = MPI_Wait(&req[i], &status[i]); - if (sub_ret != MPI_SUCCESS) { - ret = sub_ret; - } - } - return ret; + auto fs_api = Singleton::GetInstance(); + return fs_api->WaitAll(count, req, status); } + /** * Metadata functions */ int HERMES_DECL(MPI_File_open)(MPI_Comm comm, const char *filename, int amode, MPI_Info info, MPI_File *fh) { - int status; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(filename)) { LOG(INFO) << "Intercept MPI_File_open for filename: " << filename << " and mode: " << amode << " is tracked." << std::endl; - status = open_internal(comm, filename, amode, info, fh); - } else { - MAP_OR_FAIL(MPI_File_open); - status = real_MPI_File_open_(comm, filename, amode, info, fh); + AdapterStat stat; + stat.comm = comm; + stat.amode = amode; + stat.info = info; + File f = fs_api->Open(stat, filename); + (*fh) = f.mpi_fh_; + return f.mpi_status_; } - return (status); + return real_api->MPI_File_open(comm, filename, amode, info, fh); } int HERMES_DECL(MPI_File_close)(MPI_File *fh) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(fh)) { - LOG(INFO) << "Intercept MPI_File_close." << std::endl; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fh); - if (existing.second) { - MPI_Barrier(existing.first.comm); - LOG(INFO) << "File handler is opened by adapter." << std::endl; - hapi::Context ctx; - if (existing.first.ref_count == 1) { - auto filename = existing.first.st_bkid->GetName(); - bool is_file_shared = false; - int hash_total = 0, hash_num = std::hash()(filename) % 100; - MPI_Allreduce(&hash_num, &hash_total, 1, MPI_INT, MPI_SUM, - existing.first.comm); - int comm_size_call; - - MPI_Comm_size(existing.first.comm, &comm_size_call); - if (hash_total == hash_num * comm_size_call) { - is_file_shared = true; - LOG(INFO) << "File " << filename << " shared true " << std::endl; - } else { - LOG(INFO) << "File " << filename << " shared false " << hash_total - << " " << hash_num << std::endl; - } - auto persist = INTERCEPTOR_LIST->Persists(filename); - mdm->Delete(fh); - const auto &blob_names = existing.first.st_blobs; - auto blob_vbucket_vec = std::vector(); - if (!blob_names.empty() && persist) { - LOG(INFO) << "MPI-IO close Adapter flushes " << blob_names.size() - << " blobs to filename:" << filename << "." << std::endl; - auto vbucket_name = filename + "_" + std::to_string(mdm->rank); - INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); - hapi::VBucket file_vbucket(vbucket_name, mdm->GetHermes(), - ctx); - auto offset_map = std::unordered_map(); - - for (auto blob_name : blob_names) { - auto h_struct = mdm->DecodeBlobNameLocal(blob_name); - auto status = file_vbucket.Link(blob_name, filename, ctx); - if (!status.Failed()) { - if (h_struct.first == -1) { - auto page_index = std::stol(blob_name) - 1; - offset_map.emplace(blob_name, page_index * kPageSize); - } else { - auto page_index = std::stol(h_struct.second.blob_name_) - 1; - offset_map.emplace(blob_name, page_index * kPageSize + - h_struct.second.offset_); - } - } - } - bool flush_synchronously = true; - hapi::PersistTrait persist_trait(filename, offset_map, - flush_synchronously); - file_vbucket.Attach(&persist_trait, ctx); - file_vbucket.Destroy(ctx); - - for (const auto &vbucket : existing.first.st_vbuckets) { - auto blob_vbucket = - new hapi::VBucket(vbucket, mdm->GetHermes(), ctx); - auto blob_names_v = blob_vbucket->GetLinks(ctx); - for (auto &blob_name : blob_names_v) { - blob_vbucket->Unlink(blob_name, - existing.first.st_bkid->GetName()); - } - if (is_file_shared && mdm->rank % 2 == 1) - blob_vbucket->Release(ctx); - blob_vbucket_vec.push_back(blob_vbucket); - } - for (auto &blob_name : existing.first.st_blobs) { - existing.first.st_bkid->DeleteBlob(blob_name); - } - existing.first.st_blobs.clear(); - INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename); - } - MPI_Barrier(existing.first.comm); - for (const auto &blob_vbucket : blob_vbucket_vec) { - if (!is_file_shared || mdm->rank % 2 == 0) blob_vbucket->Destroy(); - delete(blob_vbucket); - } - if (is_file_shared) { - existing.first.st_bkid->Release(ctx); - } - MPI_Barrier(existing.first.comm); - existing.first.st_bkid->Destroy(ctx); - if (existing.first.a_mode & MPI_MODE_DELETE_ON_CLOSE) { - fs::remove(filename); - } + File f; f.mpi_fh_ = *fh; fs_api->_InitFile(f); + return fs_api->Close(f, stat_exists); + } + return real_api->MPI_File_close(fh); +} - } else { - LOG(INFO) << "File handler is opened by more than one open." - << std::endl; - existing.first.ref_count--; - mdm->Update(fh, existing.first); - existing.first.st_bkid->Release(ctx); - } - MPI_Barrier(existing.first.comm); - ret = MPI_SUCCESS; - } else { - MAP_OR_FAIL(MPI_File_close); - ret = real_MPI_File_close_(fh); - } - } else { - MAP_OR_FAIL(MPI_File_close); - ret = real_MPI_File_close_(fh); +int HERMES_DECL(MPI_File_seek)(MPI_File fh, MPI_Offset offset, int whence) { + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); + if (IsTracked(&fh)) { + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + return fs_api->Seek(f, stat_exists, offset, whence); } - return (ret); + return real_api->MPI_File_seek(fh, offset, whence); } int HERMES_DECL(MPI_File_seek_shared)(MPI_File fh, MPI_Offset offset, int whence) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { LOG(INFO) << "Intercept MPI_File_seek_shared offset:" << offset << " whence:" << whence << "." << std::endl; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(&fh); - MPI_Offset sum_offset; - int sum_whence; - int comm_participators; - MPI_Comm_size(existing.first.comm, &comm_participators); - MPI_Allreduce(&offset, &sum_offset, 1, MPI_LONG_LONG_INT, MPI_SUM, - existing.first.comm); - MPI_Allreduce(&whence, &sum_whence, 1, MPI_INT, MPI_SUM, - existing.first.comm); - if (sum_offset / comm_participators != offset) { - LOG(ERROR) - << "Same offset should be passed across the opened file communicator." - << std::endl; - } - if (sum_whence / comm_participators != whence) { - LOG(ERROR) - << "Same whence should be passed across the opened file communicator." - << std::endl; - } - ret = MPI_File_seek(fh, offset, whence); - } else { - MAP_OR_FAIL(MPI_File_seek_shared); - ret = real_MPI_File_seek_shared_(fh, offset, whence); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + return fs_api->SeekShared(f, stat_exists, offset, whence); } - return (ret); + return real_api->MPI_File_seek_shared(fh, offset, whence); } -int HERMES_DECL(MPI_File_seek)(MPI_File fh, MPI_Offset offset, int whence) { - int ret = -1; - if (IsTracked(&fh)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(&fh); - if (existing.second) { - LOG(INFO) << "Intercept fseek offset:" << offset << " whence:" << whence - << "." << std::endl; - if (!(existing.first.a_mode & MPI_MODE_APPEND)) { - switch (whence) { - case MPI_SEEK_SET: { - existing.first.ptr = offset; - break; - } - case MPI_SEEK_CUR: { - existing.first.ptr += offset; - break; - } - case MPI_SEEK_END: { - existing.first.ptr = existing.first.size + offset; - break; - } - default: { - // TODO(hari): throw not implemented error. - } - } - mdm->Update(&fh, existing.first); - ret = 0; - } else { - LOG(INFO) - << "File pointer not updating as file was opened in append mode." - << std::endl; - ret = -1; - } - } else { - MAP_OR_FAIL(MPI_File_seek); - ret = real_MPI_File_seek_(fh, offset, whence); - } - } else { - MAP_OR_FAIL(MPI_File_seek); - ret = real_MPI_File_seek_(fh, offset, whence); - } - return (ret); -} int HERMES_DECL(MPI_File_get_position)(MPI_File fh, MPI_Offset *offset) { - int ret = -1; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(&fh); - *offset = existing.first.ptr; - ret = MPI_SUCCESS; - } else { - MAP_OR_FAIL(MPI_File_get_position); - ret = real_MPI_File_get_position_(fh, offset); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + (*offset) = static_cast(fs_api->Tell(f, stat_exists)); + return MPI_SUCCESS; } - return ret; + return real_api->MPI_File_get_position(fh, offset); } -/** - * Sync Read/Write - */ + int HERMES_DECL(MPI_File_read_all)(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(&fh); - if (existing.second) { - MPI_Barrier(existing.first.comm); - LOG(INFO) << "Intercept MPI_File_read_all." << std::endl; - auto read_ret = - read_internal(existing, buf, count, datatype, &fh, status, true); - ret = read_ret.first; - MPI_Barrier(existing.first.comm); - } else { - MAP_OR_FAIL(MPI_File_read_all); - ret = real_MPI_File_read_all_(fh, buf, count, datatype, status); - } - } else { - MAP_OR_FAIL(MPI_File_read_all); - ret = real_MPI_File_read_all_(fh, buf, count, datatype, status); + LOG(INFO) << "Intercept MPI_File_read_all." << std::endl; + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + return fs_api->ReadAll(f, stat_exists, buf, count, datatype, status); } - return (ret); + return real_api->MPI_File_read_all(fh, buf, count, datatype, status); } int HERMES_DECL(MPI_File_read_at_all)(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - ret = MPI_File_seek(fh, offset, MPI_SEEK_SET); - if (ret == MPI_SUCCESS) { - ret = MPI_File_read_all(fh, buf, count, datatype, status); - } - } else { - MAP_OR_FAIL(MPI_File_read_at_all); - ret = real_MPI_File_read_at_all_(fh, offset, buf, count, datatype, status); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + return fs_api->ReadAll(f, stat_exists, buf, + offset, count, datatype, status); } - return ret; + return real_api->MPI_File_read_at_all(fh, offset, buf, + count, datatype, status); } int HERMES_DECL(MPI_File_read_at)(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Status *status) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - ret = MPI_File_seek(fh, offset, MPI_SEEK_SET); - if (ret == MPI_SUCCESS) { - ret = MPI_File_read(fh, buf, count, datatype, status); - } - } else { - MAP_OR_FAIL(MPI_File_read_at); - ret = real_MPI_File_read_at_(fh, offset, buf, count, datatype, status); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + return fs_api->Read(f, stat_exists, buf, offset, count, datatype, status); } - return ret; + return real_api->MPI_File_read_at(fh, offset, buf, count, datatype, status); } int HERMES_DECL(MPI_File_read)(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(&fh); - if (existing.second) { - LOG(INFO) << "Intercept MPI_File_read." << std::endl; - auto read_ret = - read_internal(existing, buf, count, datatype, &fh, status); - ret = read_ret.first; - } else { - MAP_OR_FAIL(MPI_File_read); - ret = real_MPI_File_read_(fh, buf, count, datatype, status); - } - } else { - MAP_OR_FAIL(MPI_File_read); - ret = real_MPI_File_read_(fh, buf, count, datatype, status); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + int ret = fs_api->Read(f, stat_exists, buf, count, datatype, status); + if (stat_exists) return ret; } - return (ret); + return real_api->MPI_File_read(fh, buf, count, datatype, status); } int HERMES_DECL(MPI_File_read_ordered)(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(&fh); - int total; - MPI_Scan(&count, &total, 1, MPI_INT, MPI_SUM, existing.first.comm); - MPI_Offset my_offset = total - count; - ret = MPI_File_read_at_all(fh, my_offset, buf, count, datatype, status); - } else { - MAP_OR_FAIL(MPI_File_read_ordered); - ret = real_MPI_File_read_ordered_(fh, buf, count, datatype, status); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + return fs_api->ReadOrdered(f, stat_exists, buf, count, datatype, status); } - return (ret); + return real_api->MPI_File_read_ordered(fh, buf, count, datatype, status); } int HERMES_DECL(MPI_File_read_shared)(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Status *status) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - ret = MPI_File_read(fh, buf, count, datatype, status); - } else { - MAP_OR_FAIL(MPI_File_read_shared); - ret = real_MPI_File_read_shared_(fh, buf, count, datatype, status); + LOG(INFO) << "Intercept MPI_File_read_shared." << std::endl; + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + return fs_api->Read(f, stat_exists, buf, count, datatype, status); } - return ret; + return real_api->MPI_File_read_shared(fh, buf, count, datatype, status); } int HERMES_DECL(MPI_File_write_all)(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(&fh); - if (existing.second) { - MPI_Barrier(existing.first.comm); - LOG(INFO) << "Intercept MPI_File_write." << std::endl; - auto write_ret = - write_internal(existing, buf, count, datatype, &fh, status, true); - ret = write_ret.first; - MPI_Barrier(existing.first.comm); - } else { - MAP_OR_FAIL(MPI_File_write_all); - ret = real_MPI_File_write_all_(fh, buf, count, datatype, status); - } - } else { - MAP_OR_FAIL(MPI_File_write_all); - ret = real_MPI_File_write_all_(fh, buf, count, datatype, status); + LOG(INFO) << "Intercept MPI_File_write_all." << std::endl; + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + int ret = fs_api->WriteAll(f, stat_exists, buf, count, datatype, status); + if (stat_exists) return ret; } - return (ret); + return real_api->MPI_File_write_all(fh, buf, count, datatype, status); } int HERMES_DECL(MPI_File_write_at_all)(MPI_File fh, MPI_Offset offset, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - ret = MPI_File_seek(fh, offset, MPI_SEEK_SET); - if (ret == MPI_SUCCESS) { - ret = MPI_File_write_all(fh, buf, count, datatype, status); - } - } else { - MAP_OR_FAIL(MPI_File_write_at_all); - ret = real_MPI_File_write_at_all_(fh, offset, buf, count, datatype, status); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + int ret = fs_api->WriteAll(f, stat_exists, buf, + offset, count, datatype, status); + if (stat_exists) return ret; } - return ret; + return real_api->MPI_File_write_at_all(fh, offset, buf, count, + datatype, status); } int HERMES_DECL(MPI_File_write_at)(MPI_File fh, MPI_Offset offset, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); + LOG(INFO) << "In MPI_File_write_at" << std::endl; if (IsTracked(&fh)) { - ret = MPI_File_seek(fh, offset, MPI_SEEK_SET); - if (ret == MPI_SUCCESS) { - ret = MPI_File_write(fh, buf, count, datatype, status); - } - } else { - MAP_OR_FAIL(MPI_File_write_at); - ret = real_MPI_File_write_at_(fh, offset, buf, count, datatype, status); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + return fs_api->Write(f, stat_exists, buf, offset, count, datatype, status); } - return ret; + return real_api->MPI_File_write_at(fh, offset, buf, count, datatype, status); } int HERMES_DECL(MPI_File_write)(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); + LOG(INFO) << "In MPI_File_write" << std::endl; if (IsTracked(&fh)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(&fh); - if (existing.second) { - LOG(INFO) << "Intercept MPI_File_write." << std::endl; - auto write_ret = - write_internal(existing, buf, count, datatype, &fh, status, false); - ret = write_ret.first; - } else { - MAP_OR_FAIL(MPI_File_write); - ret = real_MPI_File_write_(fh, buf, count, datatype, status); - } - } else { - MAP_OR_FAIL(MPI_File_write); - ret = real_MPI_File_write_(fh, buf, count, datatype, status); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + int ret = fs_api->Write(f, stat_exists, buf, count, datatype, status); + if (stat_exists) return ret; } - return (ret); + return real_api->MPI_File_write(fh, buf, count, datatype, status); } int HERMES_DECL(MPI_File_write_ordered)(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(&fh); - int total; - MPI_Scan(&count, &total, 1, MPI_INT, MPI_SUM, existing.first.comm); - MPI_Offset my_offset = total - count; - ret = MPI_File_write_at_all(fh, my_offset, buf, count, datatype, status); - } else { - MAP_OR_FAIL(MPI_File_write_ordered); - ret = real_MPI_File_write_ordered_(fh, buf, count, datatype, status); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + return fs_api->WriteOrdered(f, stat_exists, buf, count, datatype, status); } - return (ret); + return real_api->MPI_File_write_ordered(fh, buf, count, datatype, status); } int HERMES_DECL(MPI_File_write_shared)(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Status *status) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - ret = MPI_File_write_ordered(fh, buf, count, datatype, status); - } else { - MAP_OR_FAIL(MPI_File_write_shared); - ret = real_MPI_File_write_shared_(fh, buf, count, datatype, status); + // NOTE(llogan): originally WriteOrdered + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + return fs_api->WriteOrdered(f, stat_exists, buf, count, datatype, status); } - return ret; + return real_api->MPI_File_write_shared(fh, buf, count, datatype, status); } + /** * Async Read/Write */ int HERMES_DECL(MPI_File_iread_at)(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, MPI_Request *request) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto pool = - hermes::adapter::Singleton::GetInstance(kNumThreads); - HermesRequest *req = new HermesRequest(); - auto func = std::bind(MPI_File_read_at, fh, offset, buf, count, datatype, - &req->status); - req->return_future = pool->run(func); - auto mdm = hermes::adapter::Singleton::GetInstance(); - mdm->request_map.emplace(request, req); - ret = MPI_SUCCESS; - } else { - MAP_OR_FAIL(MPI_File_iread_at); - ret = real_MPI_File_iread_at_(fh, offset, buf, count, datatype, request); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + fs_api->ARead(f, stat_exists, buf, offset, count, datatype, request); + return MPI_SUCCESS; } - return ret; + return real_api->MPI_File_iread_at(fh, offset, buf, + count, datatype, request); } int HERMES_DECL(MPI_File_iread)(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto pool = - hermes::adapter::Singleton::GetInstance(kNumThreads); - HermesRequest *req = new HermesRequest(); - auto func = - std::bind(MPI_File_read, fh, buf, count, datatype, &req->status); - req->return_future = pool->run(func); - auto mdm = hermes::adapter::Singleton::GetInstance(); - mdm->request_map.emplace(request, req); - ret = MPI_SUCCESS; - } else { - MAP_OR_FAIL(MPI_File_iread); - ret = real_MPI_File_iread_(fh, buf, count, datatype, request); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + fs_api->ARead(f, stat_exists, buf, count, datatype, request); } - return ret; + return real_api->MPI_File_iread(fh, buf, count, datatype, request); } int HERMES_DECL(MPI_File_iread_shared)(MPI_File fh, void *buf, int count, MPI_Datatype datatype, MPI_Request *request) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto pool = - hermes::adapter::Singleton::GetInstance(kNumThreads); - HermesRequest *req = new HermesRequest(); - auto func = - std::bind(MPI_File_read_shared, fh, buf, count, datatype, &req->status); - req->return_future = pool->run(func); - auto mdm = hermes::adapter::Singleton::GetInstance(); - mdm->request_map.emplace(request, req); - ret = MPI_SUCCESS; - } else { - MAP_OR_FAIL(MPI_File_iread_shared); - ret = real_MPI_File_iread_shared_(fh, buf, count, datatype, request); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + fs_api->ARead(f, stat_exists, buf, count, datatype, request); + return MPI_SUCCESS; } - return ret; + return real_api->MPI_File_iread_shared(fh, buf, count, datatype, request); } int HERMES_DECL(MPI_File_iwrite_at)(MPI_File fh, MPI_Offset offset, const void *buf, int count, MPI_Datatype datatype, MPI_Request *request) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto pool = - hermes::adapter::Singleton::GetInstance(kNumThreads); - HermesRequest *req = new HermesRequest(); - auto func = std::bind(MPI_File_write_at, fh, offset, buf, count, datatype, - &req->status); - req->return_future = pool->run(func); - auto mdm = hermes::adapter::Singleton::GetInstance(); - mdm->request_map.emplace(request, req); - ret = MPI_SUCCESS; - } else { - MAP_OR_FAIL(MPI_File_iwrite_at); - ret = real_MPI_File_iwrite_at_(fh, offset, buf, count, datatype, request); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + fs_api->AWrite(f, stat_exists, buf, offset, count, datatype, request); + return MPI_SUCCESS; } - return ret; + return real_api->MPI_File_iwrite_at(fh, offset, buf, + count, datatype, request); } int HERMES_DECL(MPI_File_iwrite)(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Request *request) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto pool = - hermes::adapter::Singleton::GetInstance(kNumThreads); - HermesRequest *req = new HermesRequest(); - auto func = - std::bind(MPI_File_write, fh, buf, count, datatype, &req->status); - req->return_future = pool->run(func); - auto mdm = hermes::adapter::Singleton::GetInstance(); - mdm->request_map.emplace(request, req); - ret = MPI_SUCCESS; - } else { - MAP_OR_FAIL(MPI_File_iwrite); - ret = real_MPI_File_iwrite_(fh, buf, count, datatype, request); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + fs_api->AWrite(f, stat_exists, buf, count, datatype, request); + return MPI_SUCCESS; } - return ret; + return real_api->MPI_File_iwrite(fh, buf, count, datatype, request); } int HERMES_DECL(MPI_File_iwrite_shared)(MPI_File fh, const void *buf, int count, MPI_Datatype datatype, MPI_Request *request) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto pool = - hermes::adapter::Singleton::GetInstance(kNumThreads); - HermesRequest *req = new HermesRequest(); - auto func = std::bind(MPI_File_write_shared, fh, buf, count, datatype, - &req->status); - req->return_future = pool->run(func); - auto mdm = hermes::adapter::Singleton::GetInstance(); - mdm->request_map.emplace(request, req); - ret = MPI_SUCCESS; - } else { - MAP_OR_FAIL(MPI_File_iwrite_shared); - ret = real_MPI_File_iwrite_shared_(fh, buf, count, datatype, request); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + fs_api->AWriteOrdered(f, stat_exists, buf, count, datatype, request); + return MPI_SUCCESS; } - return ret; + return real_api->MPI_File_iwrite_shared(fh, buf, count, + datatype, request); } /** * Other functions */ int HERMES_DECL(MPI_File_sync)(MPI_File fh) { - int ret = -1; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (IsTracked(&fh)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(&fh); - if (existing.second) { - LOG(INFO) << "Intercept MPI_File_sync." << std::endl; - auto filename = existing.first.st_bkid->GetName(); - auto persist = INTERCEPTOR_LIST->Persists(filename); - mdm->Delete(&fh); - hapi::Context ctx; - const auto &blob_names = existing.first.st_blobs; - if (!blob_names.empty() && persist) { - LOG(INFO) << "MPI-IO sync Adapter flushes " << blob_names.size() - << " blobs to filename:" << filename << "." << std::endl; - INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); - hapi::VBucket file_vbucket(filename, mdm->GetHermes()); - auto offset_map = std::unordered_map(); - - for (auto blob_name : blob_names) { - auto h_struct = mdm->DecodeBlobNameLocal(blob_name); - auto status = file_vbucket.Link(blob_name, filename, ctx); - if (!status.Failed()) { - if (h_struct.first == -1) { - auto page_index = std::stol(blob_name) - 1; - offset_map.emplace(blob_name, page_index * kPageSize); - } else { - auto page_index = std::stol(h_struct.second.blob_name_) - 1; - offset_map.emplace( - blob_name, page_index * kPageSize + h_struct.second.offset_); - } - } - } - bool flush_synchronously = true; - hapi::PersistTrait persist_trait(filename, offset_map, - flush_synchronously); - file_vbucket.Attach(&persist_trait); - file_vbucket.Destroy(); - for (const auto &vbucket : existing.first.st_vbuckets) { - hapi::VBucket blob_vbucket(vbucket, mdm->GetHermes()); - auto blob_names_v = blob_vbucket.GetLinks(ctx); - for (auto &blob_name : blob_names_v) { - blob_vbucket.Unlink(blob_name, existing.first.st_bkid->GetName()); - } - auto blob_names_temp = blob_vbucket.GetLinks(ctx); - blob_vbucket.Destroy(); - } - for (auto &blob_name : existing.first.st_blobs) { - existing.first.st_bkid->DeleteBlob(blob_name); - } - existing.first.st_blobs.clear(); - INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename); - } - ret = 0; - } - } else { - MAP_OR_FAIL(MPI_File_sync); - ret = real_MPI_File_sync_(fh); + File f; f.mpi_fh_ = fh; fs_api->_InitFile(f); + fs_api->Sync(f, stat_exists); + return 0; } - return (ret); + return real_api->MPI_File_sync(fh); } diff --git a/adapter/mpiio/mpiio.h b/adapter/mpiio/mpiio.h deleted file mode 100644 index aab175dec..000000000 --- a/adapter/mpiio/mpiio.h +++ /dev/null @@ -1,128 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_MPIIO_H -#define HERMES_MPIIO_H - -/** - * Standard header - */ -#include -#include -#include - -#include - -/** - * Dependent library headers - */ -#include -#include - -#include - -/** - * Internal headers - */ -#include -#include -#include - -#include "adapter_utils.h" -#include "constants.h" -#include "singleton.h" -#include "interceptor.h" -#include "mpiio/mapper/mapper_factory.h" - -#include "interceptor.cc" -#include "mpiio/metadata_manager.cc" - -/** - * Function declaration - */ -HERMES_FORWARD_DECL(MPI_File_close, int, (MPI_File * fh)); -HERMES_FORWARD_DECL(MPI_File_iread_at, int, - (MPI_File fh, MPI_Offset offset, void *buf, int count, - MPI_Datatype datatype, MPI_Request *request)); -HERMES_FORWARD_DECL(MPI_File_iread, int, - (MPI_File fh, void *buf, int count, MPI_Datatype datatype, - MPI_Request *request)); -HERMES_FORWARD_DECL(MPI_File_iread_shared, int, - (MPI_File fh, void *buf, int count, MPI_Datatype datatype, - MPI_Request *request)); -HERMES_FORWARD_DECL(MPI_File_iwrite_at, int, - (MPI_File fh, MPI_Offset offset, const void *buf, int count, - MPI_Datatype datatype, MPI_Request *request)); -HERMES_FORWARD_DECL(MPI_File_iwrite, int, - (MPI_File fh, const void *buf, int count, - MPI_Datatype datatype, MPI_Request *request)); -HERMES_FORWARD_DECL(MPI_File_iwrite_shared, int, - (MPI_File fh, const void *buf, int count, - MPI_Datatype datatype, MPI_Request *request)); -HERMES_FORWARD_DECL(MPI_File_open, int, - (MPI_Comm comm, const char *filename, int amode, - MPI_Info info, MPI_File *fh)); - -HERMES_FORWARD_DECL(MPI_File_read_all, int, - (MPI_File fh, void *buf, int count, MPI_Datatype datatype, - MPI_Status *status)); -HERMES_FORWARD_DECL(MPI_File_read_at_all, int, - (MPI_File fh, MPI_Offset offset, void *buf, int count, - MPI_Datatype datatype, MPI_Status *status)); - -HERMES_FORWARD_DECL(MPI_File_read_at, int, - (MPI_File fh, MPI_Offset offset, void *buf, int count, - MPI_Datatype datatype, MPI_Status *status)); -HERMES_FORWARD_DECL(MPI_File_read, int, - (MPI_File fh, void *buf, int count, MPI_Datatype datatype, - MPI_Status *status)); - -HERMES_FORWARD_DECL(MPI_File_read_ordered, int, - (MPI_File fh, void *buf, int count, MPI_Datatype datatype, - MPI_Status *status)); -HERMES_FORWARD_DECL(MPI_File_read_shared, int, - (MPI_File fh, void *buf, int count, MPI_Datatype datatype, - MPI_Status *status)); -HERMES_FORWARD_DECL(MPI_File_sync, int, (MPI_File fh)); -HERMES_FORWARD_DECL(MPI_File_write_all, int, - (MPI_File fh, const void *buf, int count, - MPI_Datatype datatype, MPI_Status *status)); -HERMES_FORWARD_DECL(MPI_File_write_at_all, int, - (MPI_File fh, MPI_Offset offset, const void *buf, int count, - MPI_Datatype datatype, MPI_Status *status)); -HERMES_FORWARD_DECL(MPI_File_write_at, int, - (MPI_File fh, MPI_Offset offset, const void *buf, int count, - MPI_Datatype datatype, MPI_Status *status)); -HERMES_FORWARD_DECL(MPI_File_write, int, - (MPI_File fh, const void *buf, int count, - MPI_Datatype datatype, MPI_Status *status)); -HERMES_FORWARD_DECL(MPI_File_write_ordered, int, - (MPI_File fh, const void *buf, int count, - MPI_Datatype datatype, MPI_Status *status)); -HERMES_FORWARD_DECL(MPI_File_write_shared, int, - (MPI_File fh, const void *buf, int count, - MPI_Datatype datatype, MPI_Status *status)); -HERMES_FORWARD_DECL(MPI_File_seek, int, - (MPI_File fh, MPI_Offset offset, int whence)); -HERMES_FORWARD_DECL(MPI_File_seek_shared, int, - (MPI_File fh, MPI_Offset offset, int whence)); -HERMES_FORWARD_DECL(MPI_File_get_position, int, - (MPI_File fh, MPI_Offset *offset)); -/** - * MPI functions declarations - */ -HERMES_FORWARD_DECL(MPI_Init, int, (int *argc, char ***argv)); -HERMES_FORWARD_DECL(MPI_Finalize, int, (void)); -HERMES_FORWARD_DECL(MPI_Wait, int, (MPI_Request*, MPI_Status*)); -HERMES_FORWARD_DECL(MPI_Waitall, int, (int, MPI_Request*, MPI_Status*)) - -#endif // HERMES_MPIIO_H diff --git a/adapter/mpiio/real_api.h b/adapter/mpiio/real_api.h new file mode 100644 index 000000000..d95bfe2c8 --- /dev/null +++ b/adapter/mpiio/real_api.h @@ -0,0 +1,471 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_ADAPTER_MPIIO_H +#define HERMES_ADAPTER_MPIIO_H +#include +#include +#include +#include +#include +#include + +namespace hermes::adapter::mpiio { + +class API { + public: + typedef int (*MPI_Init_t)(int * argc, char *** argv); + int (*MPI_Init)(int * argc, char *** argv) = nullptr; + typedef int (*MPI_Finalize_t)( void); + int (*MPI_Finalize)( void) = nullptr; + typedef int (*MPI_Wait_t)(MPI_Request * req, MPI_Status * status); + int (*MPI_Wait)(MPI_Request * req, MPI_Status * status) = nullptr; + typedef int (*MPI_Waitall_t)(int count, MPI_Request * req, + MPI_Status * status); + int (*MPI_Waitall)(int count, MPI_Request * req, + MPI_Status * status) = nullptr; + typedef int (*MPI_File_open_t)(MPI_Comm comm, const char * filename, + int amode, MPI_Info info, MPI_File * fh); + int (*MPI_File_open)(MPI_Comm comm, const char * filename, int amode, + MPI_Info info, MPI_File * fh) = nullptr; + typedef int (*MPI_File_close_t)(MPI_File * fh); + int (*MPI_File_close)(MPI_File * fh) = nullptr; + typedef int (*MPI_File_seek_shared_t)(MPI_File fh, + MPI_Offset offset, int whence); + int (*MPI_File_seek_shared)(MPI_File fh, MPI_Offset offset, + int whence) = nullptr; + typedef int (*MPI_File_seek_t)(MPI_File fh, MPI_Offset offset, int whence); + int (*MPI_File_seek)(MPI_File fh, MPI_Offset offset, int whence) = nullptr; + typedef int (*MPI_File_get_position_t)(MPI_File fh, MPI_Offset * offset); + int (*MPI_File_get_position)(MPI_File fh, MPI_Offset * offset) = nullptr; + typedef int (*MPI_File_read_all_t)(MPI_File fh, void * buf, int count, + MPI_Datatype datatype, + MPI_Status * status); + int (*MPI_File_read_all)(MPI_File fh, void * buf, int count, + MPI_Datatype datatype, + MPI_Status * status) = nullptr; + typedef int (*MPI_File_read_at_all_t)(MPI_File fh, MPI_Offset offset, + void * buf, int count, + MPI_Datatype datatype, + MPI_Status * status); + int (*MPI_File_read_at_all)(MPI_File fh, MPI_Offset offset, void * buf, + int count, MPI_Datatype datatype, + MPI_Status * status) = nullptr; + typedef int (*MPI_File_read_at_t)(MPI_File fh, MPI_Offset offset, + void * buf, int count, + MPI_Datatype datatype, MPI_Status * status); + int (*MPI_File_read_at)(MPI_File fh, MPI_Offset offset, + void * buf, int count, MPI_Datatype datatype, + MPI_Status * status) = nullptr; + typedef int (*MPI_File_read_t)(MPI_File fh, void * buf, int count, + MPI_Datatype datatype, MPI_Status * status); + int (*MPI_File_read)(MPI_File fh, void * buf, int count, + MPI_Datatype datatype, MPI_Status * status) = nullptr; + typedef int (*MPI_File_read_ordered_t)(MPI_File fh, void * buf, int count, + MPI_Datatype datatype, + MPI_Status * status); + int (*MPI_File_read_ordered)(MPI_File fh, void * buf, int count, + MPI_Datatype datatype, + MPI_Status * status) = nullptr; + typedef int (*MPI_File_read_shared_t)(MPI_File fh, void * buf, + int count, MPI_Datatype datatype, + MPI_Status * status); + int (*MPI_File_read_shared)(MPI_File fh, void * buf, int count, + MPI_Datatype datatype, + MPI_Status * status) = nullptr; + typedef int (*MPI_File_write_all_t)(MPI_File fh, const void * buf, + int count, MPI_Datatype datatype, + MPI_Status * status); + int (*MPI_File_write_all)(MPI_File fh, const void * buf, + int count, MPI_Datatype datatype, + MPI_Status * status) = nullptr; + typedef int (*MPI_File_write_at_all_t)(MPI_File fh, MPI_Offset offset, + const void * buf, int count, + MPI_Datatype datatype, + MPI_Status * status); + int (*MPI_File_write_at_all)(MPI_File fh, MPI_Offset offset, + const void * buf, int count, + MPI_Datatype datatype, + MPI_Status * status) = nullptr; + typedef int (*MPI_File_write_at_t)(MPI_File fh, MPI_Offset offset, + const void * buf, int count, + MPI_Datatype datatype, + MPI_Status * status); + int (*MPI_File_write_at)(MPI_File fh, MPI_Offset offset, const void * buf, + int count, MPI_Datatype datatype, + MPI_Status * status) = nullptr; + typedef int (*MPI_File_write_t)(MPI_File fh, const void * buf, int count, + MPI_Datatype datatype, MPI_Status * status); + int (*MPI_File_write)(MPI_File fh, const void * buf, int count, + MPI_Datatype datatype, MPI_Status * status) = nullptr; + typedef int (*MPI_File_write_ordered_t)(MPI_File fh, const void * buf, + int count, MPI_Datatype datatype, + MPI_Status * status); + int (*MPI_File_write_ordered)(MPI_File fh, const void * buf, int count, + MPI_Datatype datatype, + MPI_Status * status) = nullptr; + typedef int (*MPI_File_write_shared_t)(MPI_File fh, const void * buf, + int count, MPI_Datatype datatype, + MPI_Status * status); + int (*MPI_File_write_shared)(MPI_File fh, const void * buf, int count, + MPI_Datatype datatype, + MPI_Status * status) = nullptr; + typedef int (*MPI_File_iread_at_t)(MPI_File fh, MPI_Offset offset, + void * buf, int count, + MPI_Datatype datatype, + MPI_Request * request); + int (*MPI_File_iread_at)(MPI_File fh, MPI_Offset offset, void * buf, + int count, MPI_Datatype datatype, + MPI_Request * request) = nullptr; + typedef int (*MPI_File_iread_t)(MPI_File fh, void * buf, int count, + MPI_Datatype datatype, MPI_Request * request); + int (*MPI_File_iread)(MPI_File fh, void * buf, int count, + MPI_Datatype datatype, MPI_Request * request) = nullptr; + typedef int (*MPI_File_iread_shared_t)(MPI_File fh, void * buf, int count, + MPI_Datatype datatype, + MPI_Request * request); + int (*MPI_File_iread_shared)(MPI_File fh, void * buf, int count, + MPI_Datatype datatype, + MPI_Request * request) = nullptr; + typedef int (*MPI_File_iwrite_at_t)(MPI_File fh, MPI_Offset offset, + const void * buf, int count, + MPI_Datatype datatype, + MPI_Request * request); + int (*MPI_File_iwrite_at)(MPI_File fh, MPI_Offset offset, const void * buf, + int count, MPI_Datatype datatype, + MPI_Request * request) = nullptr; + typedef int (*MPI_File_iwrite_t)(MPI_File fh, const void * buf, int count, + MPI_Datatype datatype, + MPI_Request * request); + int (*MPI_File_iwrite)(MPI_File fh, const void * buf, int count, + MPI_Datatype datatype, + MPI_Request * request) = nullptr; + typedef int (*MPI_File_iwrite_shared_t)(MPI_File fh, const void * buf, + int count, MPI_Datatype datatype, + MPI_Request * request); + int (*MPI_File_iwrite_shared)(MPI_File fh, const void * buf, int count, + MPI_Datatype datatype, + MPI_Request * request) = nullptr; + typedef int (*MPI_File_sync_t)(MPI_File fh); + int (*MPI_File_sync)(MPI_File fh) = nullptr; + API() { + void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, + "mpiio_intercepted"); + if (is_intercepted) { + MPI_Init = (MPI_Init_t)dlsym(RTLD_NEXT, "MPI_Init"); + } else { + MPI_Init = (MPI_Init_t)dlsym(RTLD_DEFAULT, "MPI_Init"); + } + if (MPI_Init == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_Init" << std::endl; + } + if (is_intercepted) { + MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_NEXT, + "MPI_Finalize"); + } else { + MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_DEFAULT, + "MPI_Finalize"); + } + if (MPI_Finalize == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_Finalize" << std::endl; + } + if (is_intercepted) { + MPI_Wait = (MPI_Wait_t)dlsym(RTLD_NEXT, "MPI_Wait"); + } else { + MPI_Wait = (MPI_Wait_t)dlsym(RTLD_DEFAULT, "MPI_Wait"); + } + if (MPI_Wait == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_Wait" << std::endl; + } + if (is_intercepted) { + MPI_Waitall = (MPI_Waitall_t)dlsym(RTLD_NEXT, + "MPI_Waitall"); + } else { + MPI_Waitall = (MPI_Waitall_t)dlsym(RTLD_DEFAULT, + "MPI_Waitall"); + } + if (MPI_Waitall == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_Waitall" << std::endl; + } + if (is_intercepted) { + MPI_File_open = (MPI_File_open_t)dlsym(RTLD_NEXT, + "MPI_File_open"); + } else { + MPI_File_open = (MPI_File_open_t)dlsym(RTLD_DEFAULT, + "MPI_File_open"); + } + if (MPI_File_open == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_open" << std::endl; + } + if (is_intercepted) { + MPI_File_close = (MPI_File_close_t)dlsym(RTLD_NEXT, + "MPI_File_close"); + } else { + MPI_File_close = (MPI_File_close_t)dlsym(RTLD_DEFAULT, + "MPI_File_close"); + } + if (MPI_File_close == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_close" << std::endl; + } + if (is_intercepted) { + MPI_File_seek_shared = (MPI_File_seek_shared_t) + dlsym(RTLD_NEXT, "MPI_File_seek_shared"); + } else { + MPI_File_seek_shared = (MPI_File_seek_shared_t) + dlsym(RTLD_DEFAULT, "MPI_File_seek_shared"); + } + if (MPI_File_seek_shared == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_seek_shared" << std::endl; + } + if (is_intercepted) { + MPI_File_seek = (MPI_File_seek_t) + dlsym(RTLD_NEXT, "MPI_File_seek"); + } else { + MPI_File_seek = (MPI_File_seek_t) + dlsym(RTLD_DEFAULT, "MPI_File_seek"); + } + if (MPI_File_seek == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_seek" << std::endl; + } + if (is_intercepted) { + MPI_File_get_position = (MPI_File_get_position_t) + dlsym(RTLD_NEXT, "MPI_File_get_position"); + } else { + MPI_File_get_position = (MPI_File_get_position_t) + dlsym(RTLD_DEFAULT, "MPI_File_get_position"); + } + if (MPI_File_get_position == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_get_position" << std::endl; + } + if (is_intercepted) { + MPI_File_read_all = (MPI_File_read_all_t) + dlsym(RTLD_NEXT, "MPI_File_read_all"); + } else { + MPI_File_read_all = (MPI_File_read_all_t) + dlsym(RTLD_DEFAULT, "MPI_File_read_all"); + } + if (MPI_File_read_all == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_read_all" << std::endl; + } + if (is_intercepted) { + MPI_File_read_at_all = (MPI_File_read_at_all_t) + dlsym(RTLD_NEXT, "MPI_File_read_at_all"); + } else { + MPI_File_read_at_all = (MPI_File_read_at_all_t) + dlsym(RTLD_DEFAULT, "MPI_File_read_at_all"); + } + if (MPI_File_read_at_all == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_read_at_all" << std::endl; + } + if (is_intercepted) { + MPI_File_read_at = (MPI_File_read_at_t) + dlsym(RTLD_NEXT, "MPI_File_read_at"); + } else { + MPI_File_read_at = (MPI_File_read_at_t) + dlsym(RTLD_DEFAULT, "MPI_File_read_at"); + } + if (MPI_File_read_at == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_read_at" << std::endl; + } + if (is_intercepted) { + MPI_File_read = (MPI_File_read_t) + dlsym(RTLD_NEXT, "MPI_File_read"); + } else { + MPI_File_read = (MPI_File_read_t) + dlsym(RTLD_DEFAULT, "MPI_File_read"); + } + if (MPI_File_read == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_read" << std::endl; + } + if (is_intercepted) { + MPI_File_read_ordered = (MPI_File_read_ordered_t) + dlsym(RTLD_NEXT, "MPI_File_read_ordered"); + } else { + MPI_File_read_ordered = (MPI_File_read_ordered_t) + dlsym(RTLD_DEFAULT, "MPI_File_read_ordered"); + } + if (MPI_File_read_ordered == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_read_ordered" << std::endl; + } + if (is_intercepted) { + MPI_File_read_shared = (MPI_File_read_shared_t) + dlsym(RTLD_NEXT, "MPI_File_read_shared"); + } else { + MPI_File_read_shared = (MPI_File_read_shared_t) + dlsym(RTLD_DEFAULT, "MPI_File_read_shared"); + } + if (MPI_File_read_shared == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_read_shared" << std::endl; + } + if (is_intercepted) { + MPI_File_write_all = (MPI_File_write_all_t) + dlsym(RTLD_NEXT, "MPI_File_write_all"); + } else { + MPI_File_write_all = (MPI_File_write_all_t) + dlsym(RTLD_DEFAULT, "MPI_File_write_all"); + } + if (MPI_File_write_all == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_write_all" << std::endl; + } + if (is_intercepted) { + MPI_File_write_at_all = (MPI_File_write_at_all_t) + dlsym(RTLD_NEXT, "MPI_File_write_at_all"); + } else { + MPI_File_write_at_all = (MPI_File_write_at_all_t) + dlsym(RTLD_DEFAULT, "MPI_File_write_at_all"); + } + if (MPI_File_write_at_all == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_write_at_all" << std::endl; + } + if (is_intercepted) { + MPI_File_write_at = (MPI_File_write_at_t) + dlsym(RTLD_NEXT, "MPI_File_write_at"); + } else { + MPI_File_write_at = (MPI_File_write_at_t) + dlsym(RTLD_DEFAULT, "MPI_File_write_at"); + } + if (MPI_File_write_at == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_write_at" << std::endl; + } + if (is_intercepted) { + MPI_File_write = (MPI_File_write_t) + dlsym(RTLD_NEXT, "MPI_File_write"); + } else { + MPI_File_write = (MPI_File_write_t) + dlsym(RTLD_DEFAULT, "MPI_File_write"); + } + if (MPI_File_write == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_write" << std::endl; + } + if (is_intercepted) { + MPI_File_write_ordered = (MPI_File_write_ordered_t) + dlsym(RTLD_NEXT, "MPI_File_write_ordered"); + } else { + MPI_File_write_ordered = (MPI_File_write_ordered_t) + dlsym(RTLD_DEFAULT, "MPI_File_write_ordered"); + } + if (MPI_File_write_ordered == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_write_ordered" << std::endl; + } + if (is_intercepted) { + MPI_File_write_shared = (MPI_File_write_shared_t) + dlsym(RTLD_NEXT, "MPI_File_write_shared"); + } else { + MPI_File_write_shared = (MPI_File_write_shared_t) + dlsym(RTLD_DEFAULT, "MPI_File_write_shared"); + } + if (MPI_File_write_shared == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_write_shared" << std::endl; + } + if (is_intercepted) { + MPI_File_iread_at = (MPI_File_iread_at_t) + dlsym(RTLD_NEXT, "MPI_File_iread_at"); + } else { + MPI_File_iread_at = (MPI_File_iread_at_t) + dlsym(RTLD_DEFAULT, "MPI_File_iread_at"); + } + if (MPI_File_iread_at == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_iread_at" << std::endl; + } + if (is_intercepted) { + MPI_File_iread = (MPI_File_iread_t) + dlsym(RTLD_NEXT, "MPI_File_iread"); + } else { + MPI_File_iread = (MPI_File_iread_t) + dlsym(RTLD_DEFAULT, "MPI_File_iread"); + } + if (MPI_File_iread == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_iread" << std::endl; + } + if (is_intercepted) { + MPI_File_iread_shared = (MPI_File_iread_shared_t) + dlsym(RTLD_NEXT, "MPI_File_iread_shared"); + } else { + MPI_File_iread_shared = (MPI_File_iread_shared_t) + dlsym(RTLD_DEFAULT, "MPI_File_iread_shared"); + } + if (MPI_File_iread_shared == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_iread_shared" << std::endl; + } + if (is_intercepted) { + MPI_File_iwrite_at = (MPI_File_iwrite_at_t) + dlsym(RTLD_NEXT, "MPI_File_iwrite_at"); + } else { + MPI_File_iwrite_at = (MPI_File_iwrite_at_t) + dlsym(RTLD_DEFAULT, "MPI_File_iwrite_at"); + } + if (MPI_File_iwrite_at == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_iwrite_at" << std::endl; + } + if (is_intercepted) { + MPI_File_iwrite = (MPI_File_iwrite_t) + dlsym(RTLD_NEXT, "MPI_File_iwrite"); + } else { + MPI_File_iwrite = (MPI_File_iwrite_t) + dlsym(RTLD_DEFAULT, "MPI_File_iwrite"); + } + if (MPI_File_iwrite == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_iwrite" << std::endl; + } + if (is_intercepted) { + MPI_File_iwrite_shared = (MPI_File_iwrite_shared_t) + dlsym(RTLD_NEXT, "MPI_File_iwrite_shared"); + } else { + MPI_File_iwrite_shared = (MPI_File_iwrite_shared_t) + dlsym(RTLD_DEFAULT, "MPI_File_iwrite_shared"); + } + if (MPI_File_iwrite_shared == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_iwrite_shared" << std::endl; + } + if (is_intercepted) { + MPI_File_sync = (MPI_File_sync_t) + dlsym(RTLD_NEXT, "MPI_File_sync"); + } else { + MPI_File_sync = (MPI_File_sync_t) + dlsym(RTLD_DEFAULT, "MPI_File_sync"); + } + if (MPI_File_sync == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_File_sync" << std::endl; + } + } +}; +} // namespace hermes::adapter::mpiio + +#endif // HERMES_ADAPTER_MPIIO_H diff --git a/adapter/posix/CMakeLists.txt b/adapter/posix/CMakeLists.txt index de9ac18d8..edc46e87c 100644 --- a/adapter/posix/CMakeLists.txt +++ b/adapter/posix/CMakeLists.txt @@ -1,4 +1,5 @@ project(PosixAdapter VERSION ${HERMES_PACKAGE_VERSION}) +include_directories(${HERMES_ADAPTER_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) # POSIX src code. We only include posix.cc as it includes other cc to reduce compilation time. set(POSIX_ADAPTER_SRC posix.cc) @@ -6,25 +7,32 @@ set(POSIX_ADAPTER_SRC posix.cc) set(HERMES_POSIX_ADAPTER_DIR ${HERMES_ADAPTER_DIR}/posix) # Only posix.h is the public adapter. -set(POSIX_ADAPTER_PUBLIC_HEADER ${HERMES_POSIX_ADAPTER_DIR}/posix.h) -# Private headers -set(POSIX_ADAPTER_PRIVATE_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/metadata_manager.h - ${CMAKE_CURRENT_SOURCE_DIR}/mapper/mapper_factory.h - ${CMAKE_CURRENT_SOURCE_DIR}/mapper/abstract_mapper.h - ${CMAKE_CURRENT_SOURCE_DIR}/mapper/balanced_mapper.h - ${CMAKE_CURRENT_SOURCE_DIR}/datastructures.h - ${CMAKE_CURRENT_SOURCE_DIR}/enumerations.h - ${CMAKE_CURRENT_SOURCE_DIR}/constants.h) +set(POSIX_ADAPTER_PUBLIC_HEADER + ${HERMES_POSIX_ADAPTER_DIR}/real_api.h + ${HERMES_POSIX_ADAPTER_DIR}/fs_api.h) + +# Add library hermes_posix_backend +add_library(hermes_posix_backend ${CMAKE_CURRENT_SOURCE_DIR}/fs_api.cc) +add_dependencies(hermes_posix_backend hermes) +target_link_libraries(hermes_posix_backend hermes MPI::MPI_CXX glog::glog stdc++fs dl) # Add library hermes_posix -add_library(hermes_posix SHARED ${POSIX_ADAPTER_PRIVATE_HEADER} ${POSIX_ADAPTER_PUBLIC_HEADER} ${POSIX_ADAPTER_SRC}) -target_include_directories(hermes_posix PRIVATE ${HERMES_ADAPTER_DIR}) -add_dependencies(hermes_posix hermes) -target_link_libraries(hermes_posix hermes MPI::MPI_CXX glog::glog stdc++fs) +add_library(hermes_posix SHARED ${POSIX_ADAPTER_PUBLIC_HEADER} ${POSIX_ADAPTER_SRC}) +add_dependencies(hermes_posix hermes_posix_backend) +target_link_libraries(hermes_posix hermes_posix_backend) #----------------------------------------------------------------------------- # Add Target(s) to CMake Install #----------------------------------------------------------------------------- +install( + TARGETS + hermes_posix_backend + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} +) install( TARGETS hermes_posix diff --git a/adapter/posix/constants.h b/adapter/posix/constants.h deleted file mode 100644 index 98f80d993..000000000 --- a/adapter/posix/constants.h +++ /dev/null @@ -1,44 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_POSIX_COMMON_CONSTANTS_H -#define HERMES_POSIX_COMMON_CONSTANTS_H - -/** - * Standard header - */ - -/** - * Dependent library header - */ - -/** - * Internal header - */ -#include "posix/enumerations.h" - -/** - * Constants file for POSIX adapter. - */ -using hermes::adapter::posix::MapperType; - -/** - * Which mapper to be used by POSIX adapter. - */ -const MapperType kMapperType = MapperType::BALANCED; - -/** - * String delimiter - */ -const char kStringDelimiter = '#'; - -#endif // HERMES_POSIX_COMMON_CONSTANTS_H diff --git a/adapter/posix/datastructures.h b/adapter/posix/datastructures.h deleted file mode 100644 index 11d177c2d..000000000 --- a/adapter/posix/datastructures.h +++ /dev/null @@ -1,234 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_POSIX_ADAPTER_DATASTRUCTURES_H -#define HERMES_POSIX_ADAPTER_DATASTRUCTURES_H - -/** - * Standard header - */ -#include -#include -#include - -/** - * Dependent library header - */ - -/** - * Internal header - */ -#include -#include -#include - -/** - * Namespace simplification. - */ -namespace hapi = hermes::api; - -namespace hermes::adapter::posix { - -/** - * FileID structure used as an identifier in POSIX adapter. - */ -struct FileID { - /** - * attributes - */ - dev_t dev_id_; // device id to place the file. - ino_t inode_num_; // inode number refering to the file. - /** - * Constructor - */ - FileID() : dev_id_(), inode_num_() {} /* default constructor */ - FileID(dev_t dev_id, ino_t inode_num) - : dev_id_(dev_id), - inode_num_(inode_num) {} /* parameterized constructor */ - FileID(const FileID &other) - : dev_id_(other.dev_id_), - inode_num_(other.inode_num_) {} /* copy constructor*/ - FileID(FileID &&other) - : dev_id_(other.dev_id_), - inode_num_(other.inode_num_) {} /* move constructor*/ - - /** - * Operators defined - */ - /* Assignment operator. */ - FileID &operator=(const FileID &other) { - dev_id_ = other.dev_id_; - inode_num_ = other.inode_num_; - return *this; - } - - /* Equal operator. */ - bool operator==(const FileID &o) const { - return dev_id_ == o.dev_id_ && inode_num_ == o.inode_num_; - } -}; - -/** - * Structure POSIX adapter uses to define a file state. - */ -struct FileStruct { - /** - * attributes - */ - FileID file_id_; // fileID to identify a file uniquely. - size_t offset_; // file pointer within the file. - size_t size_; // size of data refered in file. - /** - * Constructor - */ - FileStruct() : file_id_(), offset_(0), size_(0) {} /* default constructor */ - FileStruct(FileID file_id, size_t offset, size_t size) - : file_id_(file_id), - offset_(offset), - size_(size) {} /* parameterized constructor */ - FileStruct(const FileStruct &other) - : file_id_(other.file_id_), - offset_(other.offset_), - size_(other.size_) {} /* copy constructor*/ - FileStruct(FileStruct &&other) - : file_id_(other.file_id_), - offset_(other.offset_), - size_(other.size_) {} /* move constructor*/ - /** - * Operators defined - */ - /* Assignment operator. */ - FileStruct &operator=(const FileStruct &other) { - file_id_ = other.file_id_; - offset_ = other.offset_; - size_ = other.size_; - return *this; - } -}; - -/** - * Structure POSIX adapter uses to define Hermes blob. - */ -struct HermesStruct { - /** - * attributes - */ - std::string blob_name_; - size_t offset_; - size_t size_; - /** - * Constructor - */ - HermesStruct() - : blob_name_(), offset_(0), size_(0) {} /* default constructor */ - HermesStruct(const HermesStruct &other) - : blob_name_(other.blob_name_), - offset_(other.offset_), - size_(other.size_) {} /* copy constructor*/ - HermesStruct(HermesStruct &&other) - : blob_name_(other.blob_name_), - offset_(other.offset_), - size_(other.size_) {} /* move constructor*/ - /** - * Operators defined - */ - /* Assignment operator. */ - HermesStruct &operator=(const HermesStruct &other) { - blob_name_ = other.blob_name_; - offset_ = other.offset_; - size_ = other.size_; - return *this; - } -}; - -/** - * Stat which defines File within POSIX Adapter. - */ -struct AdapterStat { - /** - * attributes - */ - std::shared_ptr st_bkid; /* bucket associated with the file */ - std::set - st_blobs; /* Blobs access in the bucket */ - i32 ref_count; /* # of time process opens a file */ - mode_t st_mode; /* protection */ - uid_t st_uid; /* user ID of owner */ - gid_t st_gid; /* group ID of owner */ - off_t st_size; /* total size, in bytes */ - off_t st_ptr; /* Current ptr of FILE */ - blksize_t st_blksize; /* blocksize for blob within bucket */ - timespec st_atim; /* time of last access */ - timespec st_mtim; /* time of last modification */ - timespec st_ctim; /* time of last status change */ - /** - * Constructor - */ - AdapterStat() - : st_bkid(), - st_blobs(CompareBlobs), - ref_count(), - st_mode(), - st_uid(), - st_gid(), - st_size(0), - st_ptr(0), - st_blksize(4096), - st_atim(), - st_mtim(), - st_ctim() {} /* default constructor */ - explicit AdapterStat(const struct stat &st) - : st_bkid(), - st_blobs(CompareBlobs), - ref_count(1), - st_mode(st.st_mode), - st_uid(st.st_uid), - st_gid(st.st_gid), - st_size(st.st_size), - st_ptr(0), - st_blksize(st.st_blksize), - st_atim(st.st_atim), - st_mtim(st.st_mtim), - st_ctim(st.st_ctim) {} /* parameterized constructor */ - - /** - * Comparator for comparing two blobs. - */ - static bool CompareBlobs(const std::string &a, const std::string &b) { - return std::stol(a) < std::stol(b); - } -}; - -} // namespace hermes::adapter::posix - -/** - * Define hash functions for POSIX Adapter. - */ -namespace std { - -/** - * hash for FileID. - */ -template <> -struct hash { - std::size_t operator()(const hermes::adapter::posix::FileID &key) const { - std::size_t result = 0; - std::size_t h1 = std::hash{}(key.dev_id_); - std::size_t h2 = std::hash{}(key.inode_num_); - result = h1 ^ (h2 << 1); - - return result; - } -}; -} // namespace std - -#endif // HERMES_POSIX_ADAPTER_DATASTRUCTURES_H diff --git a/adapter/posix/enumerations.h b/adapter/posix/enumerations.h deleted file mode 100644 index 17198dc91..000000000 --- a/adapter/posix/enumerations.h +++ /dev/null @@ -1,25 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_POSIX_COMMON_ENUMERATIONS_H -#define HERMES_POSIX_COMMON_ENUMERATIONS_H -/** - * Enumeration for POSIX adapter. - */ -namespace hermes::adapter::posix { -/** - * Define different types of mappers supported by POSIX Adapter. - * Also define its construction in the MapperFactory. - */ -enum MapperType { BALANCED = 0 /* Balanced Mapping */ }; -} // namespace hermes::adapter::posix -#endif // HERMES_POSIX_COMMON_ENUMERATIONS_H diff --git a/adapter/posix/fs_api.cc b/adapter/posix/fs_api.cc new file mode 100644 index 000000000..8cf6c59e1 --- /dev/null +++ b/adapter/posix/fs_api.cc @@ -0,0 +1,104 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include "fs_api.h" +#include "real_api.h" + +#include "filesystem/metadata_manager.cc" +#include "filesystem/filesystem.cc" + +namespace hermes::adapter::posix { + +File PosixFS::_RealOpen(AdapterStat &stat, const std::string &path) { + File f; + if (stat.flags & O_CREAT || stat.flags & O_TMPFILE) { + f.fd_ = real_api->open(path.c_str(), stat.flags, stat.st_mode); + } else { + f.fd_ = real_api->open(path.c_str(), stat.flags); + } + if (f.fd_ < 0) { + f.status_ = false; + } + _InitFile(f); + return f; +} + +void PosixFS::_InitFile(File &f) { + struct stat st; + real_api->__fxstat(_STAT_VER, f.fd_, &st); + f.st_dev = st.st_dev; + f.st_ino = st.st_ino; +} + +void PosixFS::_OpenInitStats(File &f, AdapterStat &stat, bool bucket_exists) { + (void) bucket_exists; + struct stat st; + real_api->__fxstat(_STAT_VER, f.fd_, &st); + stat.st_mode = st.st_mode; + stat.st_uid = st.st_uid; + stat.st_gid = st.st_gid; + stat.st_size = st.st_size; + stat.st_blksize = st.st_blksize; + stat.st_atim = st.st_atim; + stat.st_mtim = st.st_mtim; + stat.st_ctim = st.st_ctim; + /*if (bucket_exists) { + stat.st_size = stat.st_bkid->GetTotalBlobSize(); + LOG(INFO) << "Since bucket exists, should reset its size to: " << stat.st_size + << std::endl; + }*/ + if (stat.flags & O_APPEND) { + stat.st_ptr = stat.st_size; + stat.is_append = true; + } +} + +size_t PosixFS::_RealWrite(const std::string &filename, off_t offset, + size_t size, const u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) { + (void) opts; (void) io_status; + LOG(INFO) << "Writing to file: " << filename + << " offset: " << offset + << " size:" << size << "." + << " file_size:" << stdfs::file_size(filename) << std::endl; + int fd = real_api->open(filename.c_str(), O_RDWR | O_CREAT); + if (fd < 0) { return 0; } + size_t write_size = real_api->pwrite(fd, data_ptr, size, offset); + real_api->close(fd); + return write_size; +} + +size_t PosixFS::_RealRead(const std::string &filename, off_t offset, + size_t size, u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) { + (void) opts; (void) io_status; + LOG(INFO) << "Read called for filename from destination: " << filename + << " on offset: " << offset + << " and size: " << size << "." + << " file_size:" << stdfs::file_size(filename) << std::endl; + int fd = real_api->open(filename.c_str(), O_RDONLY); + if (fd < 0) { return 0; } + size_t read_size = real_api->pread(fd, data_ptr, size, offset); + real_api->close(fd); + return read_size; +} + +int PosixFS::_RealSync(File &f) { + return real_api->fsync(f.fd_); +} + +int PosixFS::_RealClose(File &f) { + return real_api->close(f.fd_); +} + +} // namespace hermes::adapter::posix diff --git a/adapter/posix/fs_api.h b/adapter/posix/fs_api.h new file mode 100644 index 000000000..4e9ca7733 --- /dev/null +++ b/adapter/posix/fs_api.h @@ -0,0 +1,56 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_ADAPTER_POSIX_NATIVE_H_ +#define HERMES_ADAPTER_POSIX_NATIVE_H_ + +#include +#include "filesystem/filesystem.h" +#include "filesystem/metadata_manager.h" +#include "real_api.h" + +using hermes::adapter::fs::AdapterStat; +using hermes::adapter::fs::File; +using hermes::adapter::Singleton; +using hermes::adapter::posix::API; +using hermes::adapter::fs::IoOptions; +using hermes::adapter::fs::IoStatus; + +namespace hermes::adapter::posix { + +class PosixFS : public hermes::adapter::fs::Filesystem { + private: + API* real_api; + public: + PosixFS() { + real_api = Singleton::GetInstance(); + } + ~PosixFS() = default; + + void _InitFile(File &f) override; + + private: + void _OpenInitStats(File &f, AdapterStat &stat, bool bucket_exists) override; + File _RealOpen(AdapterStat &stat, const std::string &path) override; + size_t _RealWrite(const std::string &filename, off_t offset, size_t size, + const u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) override; + size_t _RealRead(const std::string &filename, off_t offset, size_t size, + u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) override; + int _RealSync(File &f) override; + int _RealClose(File &f) override; +}; + +} // namespace hermes::adapter::posix + +#endif // HERMES_ADAPTER_POSIX_NATIVE_H_ diff --git a/adapter/posix/mapper/abstract_mapper.h b/adapter/posix/mapper/abstract_mapper.h deleted file mode 100644 index 2623529c6..000000000 --- a/adapter/posix/mapper/abstract_mapper.h +++ /dev/null @@ -1,45 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -// -// Created by manihariharan on 12/23/20. -// - -#ifndef HERMES_ABSTRACT_ADAPTER_H -#define HERMES_ABSTRACT_ADAPTER_H - -#include "posix/datastructures.h" - -/** - * Typedef to simplify the return types - */ -typedef std::vector> - MapperReturnType; - -namespace hermes::adapter::posix { -/** - * Interface to define a mapper. - */ -class AbstractMapper { - public: - /** - * This method maps the current Operation to Hermes data structures. - * - * @param file_op, FileStruct, operations for which we are mapping. - * @return a map of FileStruct to Hermes Struct - */ - virtual MapperReturnType map(const FileStruct& file_op) = 0; -}; -} // namespace hermes::adapter::posix - -#endif // HERMES_ABSTRACT_ADAPTER_H diff --git a/adapter/posix/mapper/balanced_mapper.cc b/adapter/posix/mapper/balanced_mapper.cc deleted file mode 100644 index 206b6111f..000000000 --- a/adapter/posix/mapper/balanced_mapper.cc +++ /dev/null @@ -1,46 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "balanced_mapper.h" - -/** - * Namespace declaration for cleaner code. - */ -using hermes::adapter::posix::BalancedMapper; -using hermes::adapter::posix::FileStruct; -using hermes::adapter::posix::HermesStruct; - -MapperReturnType BalancedMapper::map(const FileStruct& file_op) { - VLOG(1) << "Mapping File with offset:" << file_op.offset_ - << " and size:" << file_op.size_ << "." << std::endl; - - auto mapper_return = MapperReturnType(); - size_t size_mapped = 0; - while (file_op.size_ > size_mapped) { - FileStruct file; - file.file_id_ = file_op.file_id_; - HermesStruct hermes; - file.offset_ = file_op.offset_ + size_mapped; - size_t page_index = file.offset_ / kPageSize; - hermes.offset_ = file.offset_ % kPageSize; - auto left_size_page = kPageSize - hermes.offset_; - hermes.size_ = left_size_page < file_op.size_ - size_mapped - ? left_size_page - : file_op.size_ - size_mapped; - - file.size_ = hermes.size_; - hermes.blob_name_ = std::to_string(page_index + 1); - mapper_return.emplace_back(file, hermes); - size_mapped += hermes.size_; - } - return mapper_return; -} diff --git a/adapter/posix/mapper/balanced_mapper.h b/adapter/posix/mapper/balanced_mapper.h deleted file mode 100644 index f2f4da321..000000000 --- a/adapter/posix/mapper/balanced_mapper.h +++ /dev/null @@ -1,38 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_BALANCED_MAPPER_H -#define HERMES_BALANCED_MAPPER_H - -#include - -#include "posix/constants.h" - -#include "abstract_mapper.h" - -namespace hermes::adapter::posix { -/** - * Implement balanced mapping - */ -class BalancedMapper : public AbstractMapper { - public: - /** - * This method maps the current Operation to Hermes data structures. - * - * @param file_op, FileStruct, operations for which we are mapping. - * @return a map of FileStruct to Hermes Struct - */ - MapperReturnType map(const FileStruct& file_op) override; -}; -} // namespace hermes::adapter::posix - -#endif // HERMES_BALANCED_MAPPER_H diff --git a/adapter/posix/posix.cc b/adapter/posix/posix.cc index 7db8dfc8c..94cb71e6a 100644 --- a/adapter/posix/posix.cc +++ b/adapter/posix/posix.cc @@ -10,381 +10,44 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "posix/posix.h" +// Dynamically checked to see which are the real APIs and which are intercepted +bool posix_intercepted = true; -#include +#include +#include +#include +#include #include "interceptor.cc" -#include "adapter_utils.cc" -#include "posix/mapper/balanced_mapper.cc" -#include "posix/metadata_manager.cc" -using hermes::adapter::posix::AdapterStat; -using hermes::adapter::posix::FileStruct; -using hermes::adapter::posix::MapperFactory; -using hermes::adapter::posix::MetadataManager; -using hermes::adapter::WeaklyCanonical; -using hermes::adapter::ReadGap; - -namespace hapi = hermes::api; -namespace fs = std::experimental::filesystem; - -using hermes::u8; +#include "interceptor.h" +#include "singleton.h" -/** - * Internal Functions - */ -size_t perform_file_write(const std::string &filename, off_t offset, - size_t size, u8 *data_ptr) { - LOG(INFO) << "Writing to file: " << filename << " offset: " << offset - << " of size:" << size << "." << std::endl; - INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); - int fd = open(filename.c_str(), O_RDWR | O_CREAT); - size_t write_size = 0; - if (fd != -1) { - auto status = lseek(fd, offset, SEEK_SET); - if (status == offset) { - write_size = write(fd, data_ptr, size); - status = close(fd); - } - } - INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename); - return write_size; -} - -size_t perform_file_read(const char *filename, off_t file_offset, void *ptr, - size_t ptr_offset, size_t size) { - LOG(INFO) << "Read called for filename from destination: " << filename - << " on offset: " << file_offset << " and size: " << size - << std::endl; - INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); - int fd = open(filename, O_RDONLY); - size_t read_size = 0; - if (fd != -1) { - int status = lseek(fd, file_offset, SEEK_SET); - if (status == file_offset) { - read_size = read(fd, (char *)ptr + ptr_offset, size); - } - status = close(fd); - } - INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename); - return read_size; -} -int simple_open(int ret, const std::string &user_path, int flags) { - std::string path_str = WeaklyCanonical(user_path).string(); - - LOG(INFO) << "Open file for filename " << path_str << " with flags " << flags - << std::endl; - auto mdm = hermes::adapter::Singleton::GetInstance(); - if (!ret) { - return ret; - } else { - auto existing = mdm->Find(ret); - if (!existing.second) { - LOG(INFO) << "File not opened before by adapter" << std::endl; - struct stat st; - MAP_OR_FAIL(__fxstat); - int status = real___fxstat_(_STAT_VER, ret, &st); - if (status == 0) { - AdapterStat stat(st); - stat.ref_count = 1; - struct timespec ts; - timespec_get(&ts, TIME_UTC); - stat.st_atim = ts; - stat.st_mtim = ts; - stat.st_ctim = ts; +#include "posix/real_api.h" +#include "posix/fs_api.h" - /* FIXME(hari) check if this initialization is correct. */ - mdm->InitializeHermes(); - - bool bucket_exists = mdm->GetHermes()->BucketExists(path_str); - // TODO(hari) how to pass to hermes to make a private bucket - stat.st_bkid = - std::make_shared(path_str, mdm->GetHermes()); - - if (bucket_exists) { - stat.st_size = stat.st_bkid->GetTotalBlobSize(); - } - - if (flags & O_APPEND) { - stat.st_ptr = stat.st_size; - } - - mdm->Create(ret, stat); - } else { - // TODO(hari): @error_handling invalid fh. - ret = -1; - } - } else { - LOG(INFO) << "File opened before by adapter" << std::endl; - existing.first.ref_count++; - struct timespec ts; - timespec_get(&ts, TIME_UTC); - existing.first.st_atim = ts; - existing.first.st_ctim = ts; - mdm->Update(ret, existing.first); - } - } - return ret; -} - -int open_internal(const std::string &path_str, int flags, int mode) { - int ret; - MAP_OR_FAIL(open); - if (flags & O_CREAT || flags & O_TMPFILE) { - ret = real_open_(path_str.c_str(), flags, mode); - } else { - ret = real_open_(path_str.c_str(), flags); - } - ret = simple_open(ret, path_str.c_str(), flags); - return ret; -} - -void PutWithPosixFallback(AdapterStat &stat, const std::string &blob_name, - const std::string &filename, u8 *data, size_t size, - size_t offset) { - hapi::Context ctx; - const char *hermes_write_only = getenv(kHermesWriteOnlyVar); - - if (hermes_write_only && hermes_write_only[0] == '1') { - // Custom DPE for write-only apps like VPIC - ctx.rr_retry = true; - ctx.disable_swap = true; - } +#ifndef O_TMPFILE +#define O_TMPFILE 0 +#endif - hapi::Status status = stat.st_bkid->Put(blob_name, data, size, ctx); - if (status.Failed()) { - LOG(WARNING) << "Failed to Put Blob " << blob_name << " to Bucket " - << filename << ". Falling back to posix I/O." << std::endl; - perform_file_write(filename, offset, size, data); - } else { - stat.st_blobs.emplace(blob_name); - } -} - -size_t write_internal(AdapterStat &stat, const void *ptr, size_t total_size, - int fp) { - std::shared_ptr bkt = stat.st_bkid; - std::string filename = bkt->GetName(); - - LOG(INFO) << "Write called for filename: " << filename << " on offset: " - << stat.st_ptr << " and size: " << total_size << std::endl; - - size_t ret = 0; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto mapper = MapperFactory().Get(kMapperType); - auto mapping = mapper->map( - FileStruct(mdm->Convert(fp), stat.st_ptr, total_size)); - size_t data_offset = 0; - LOG(INFO) << "Mapping for write has " << mapping.size() << " mappings.\n"; - - for (const auto &[finfo, hinfo] : mapping) { - auto index = std::stol(hinfo.blob_name_) - 1; - size_t offset = index * kPageSize; - auto blob_exists = bkt->ContainsBlob(hinfo.blob_name_); - u8 *put_data_ptr = (u8 *)ptr + data_offset; - size_t put_data_ptr_size = finfo.size_; - - if (!blob_exists || hinfo.size_ == kPageSize) { - LOG(INFO) << "Create or Overwrite blob " << hinfo.blob_name_ - << " of size:" << hinfo.size_ << "." << std::endl; - if (hinfo.size_ == kPageSize) { - PutWithPosixFallback(stat, hinfo.blob_name_, filename, put_data_ptr, - put_data_ptr_size, finfo.offset_); - } else if (hinfo.offset_ == 0) { - PutWithPosixFallback(stat, hinfo.blob_name_, filename, put_data_ptr, - put_data_ptr_size, offset); - } else { - hapi::Blob final_data(hinfo.offset_ + hinfo.size_); - - ReadGap(filename, offset, final_data.data(), hinfo.offset_, - hinfo.offset_); - memcpy(final_data.data() + hinfo.offset_, put_data_ptr, - put_data_ptr_size); - PutWithPosixFallback(stat, hinfo.blob_name_, filename, - final_data.data(), final_data.size(), offset); - } - } else { - LOG(INFO) << "Blob " << hinfo.blob_name_ - << " of size:" << hinfo.size_ << " exists." << std::endl; - hapi::Blob temp(0); - auto existing_blob_size = bkt->Get(hinfo.blob_name_, temp); - if (hinfo.offset_ == 0) { - LOG(INFO) << "Blob offset is 0" << std::endl; - if (hinfo.size_ >= existing_blob_size) { - LOG(INFO) << "Overwrite blob " << hinfo.blob_name_ - << " of size:" << hinfo.size_ << "." << std::endl; - PutWithPosixFallback(stat, hinfo.blob_name_, filename, - put_data_ptr, put_data_ptr_size, offset); - } else { - LOG(INFO) << "Update blob " << hinfo.blob_name_ - << " of size:" << existing_blob_size << "." << std::endl; - hapi::Blob existing_data(existing_blob_size); - bkt->Get(hinfo.blob_name_, existing_data); - memcpy(existing_data.data(), put_data_ptr, put_data_ptr_size); - - PutWithPosixFallback(stat, hinfo.blob_name_, filename, - existing_data.data(), existing_data.size(), - offset); - } - } else { - LOG(INFO) << "Blob offset: " << hinfo.offset_ << "." << std::endl; - auto new_size = hinfo.offset_ + hinfo.size_; - hapi::Blob existing_data(existing_blob_size); - bkt->Get(hinfo.blob_name_, existing_data); - bkt->DeleteBlob(hinfo.blob_name_); - if (new_size < existing_blob_size) { - new_size = existing_blob_size; - } - hapi::Blob final_data(new_size); - auto existing_data_cp_size = existing_data.size() >= hinfo.offset_ - ? hinfo.offset_ : existing_data.size(); - memcpy(final_data.data(), existing_data.data(), existing_data_cp_size); - - if (existing_blob_size < hinfo.offset_ + 1) { - ReadGap(filename, offset + existing_data_cp_size, - final_data.data() + existing_data_cp_size, - hinfo.offset_ - existing_blob_size, - hinfo.offset_ + hinfo.size_); - } - memcpy(final_data.data() + hinfo.offset_, put_data_ptr, - put_data_ptr_size); - - if (hinfo.offset_ + hinfo.size_ < existing_blob_size) { - LOG(INFO) << "Retain last portion of blob as Blob is bigger than the " - "update." << std::endl; - auto off_t = hinfo.offset_ + hinfo.size_; - memcpy(final_data.data() + off_t, existing_data.data() + off_t, - existing_blob_size - off_t); - } - - PutWithPosixFallback(stat, hinfo.blob_name_, filename, - final_data.data(), final_data.size(), offset); - } - } - data_offset += finfo.size_; - - // TODO(chogan): - // if (PersistEagerly(filename)) { - // hapi::Trait *trait = stat.st_vbkt->GetTrait(hapi::TraitType::PERSIST); - // if (trait) { - // hapi::PersistTrait *persist_trait = (hapi::PersistTrait *)trait; - // persist_trait->file_mapping.offset_map.emplace(hinfo.blob_name_, - // offset); - // } - - // stat.st_vbkt->Link(hinfo.blob_name_, filename); - // } - } - stat.st_ptr += data_offset; - stat.st_size = stat.st_size >= stat.st_ptr ? stat.st_size : stat.st_ptr; - - struct timespec ts; - timespec_get(&ts, TIME_UTC); - stat.st_mtim = ts; - stat.st_ctim = ts; - mdm->Update(fp, stat); - ret = data_offset; +using hermes::adapter::WeaklyCanonical; +using hermes::adapter::posix::API; +using hermes::adapter::posix::PosixFS; +using hermes::adapter::Singleton; +using hermes::adapter::fs::MetadataManager; +using hermes::adapter::fs::SeekMode; - return ret; -} +namespace hapi = hermes::api; +namespace stdfs = std::experimental::filesystem; +using hermes::u8; -size_t read_internal(std::pair &existing, void *ptr, - size_t total_size, int fd) { - LOG(INFO) << "Read called for filename: " << existing.first.st_bkid->GetName() - << " on offset: " << existing.first.st_ptr - << " and size: " << total_size << std::endl; - if (existing.first.st_ptr >= existing.first.st_size) return 0; - size_t ret; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto mapper = MapperFactory().Get(kMapperType); - auto mapping = mapper->map( - FileStruct(mdm->Convert(fd), existing.first.st_ptr, total_size)); - size_t total_read_size = 0; - auto filename = existing.first.st_bkid->GetName(); - LOG(INFO) << "Mapping for read has " << mapping.size() << " mapping." - << std::endl; - for (const auto &item : mapping) { - hapi::Context ctx; - auto blob_exists = - existing.first.st_bkid->ContainsBlob(item.second.blob_name_); - hapi::Blob read_data(0); - size_t read_size = 0; - if (blob_exists) { - LOG(INFO) << "Blob exists and need to read from Hermes from blob: " - << item.second.blob_name_ << "." << std::endl; - auto exiting_blob_size = - existing.first.st_bkid->Get(item.second.blob_name_, read_data, ctx); - read_data.resize(exiting_blob_size); - existing.first.st_bkid->Get(item.second.blob_name_, read_data, ctx); - bool contains_blob = exiting_blob_size > item.second.offset_; - if (contains_blob) { - read_size = read_data.size() < item.second.offset_ + item.second.size_ - ? exiting_blob_size - item.second.offset_ - : item.second.size_; - LOG(INFO) << "Blob have data and need to read from hemes " - "blob: " - << item.second.blob_name_ << " offset:" << item.second.offset_ - << " size:" << read_size << "." << std::endl; - memcpy((char *)ptr + total_read_size, - read_data.data() + item.second.offset_, read_size); - if (read_size < item.second.size_) { - contains_blob = true; - } else { - contains_blob = false; - } - } else { - LOG(INFO) << "Blob does not have data and need to read from original " - "filename: " - << filename << " offset:" << item.first.offset_ - << " size:" << item.first.size_ << "." << std::endl; - auto file_read_size = - perform_file_read(filename.c_str(), item.first.offset_, ptr, - total_read_size, item.second.size_); - read_size += file_read_size; - } - if (contains_blob && fs::exists(filename) && - fs::file_size(filename) >= item.first.offset_ + item.first.size_) { - LOG(INFO) << "Blob does not have data and need to read from original " - "filename: " - << filename << " offset:" << item.first.offset_ + read_size - << " size:" << item.second.size_ - read_size << "." - << std::endl; - auto new_read_size = perform_file_read( - filename.c_str(), item.first.offset_, ptr, - total_read_size + read_size, item.second.size_ - read_size); - read_size += new_read_size; - } - } else if (fs::exists(filename) && - fs::file_size(filename) >= - item.first.offset_ + item.first.size_) { - LOG(INFO) - << "Blob does not exists and need to read from original filename: " - << filename << " offset:" << item.first.offset_ - << " size:" << item.first.size_ << "." << std::endl; - read_size = perform_file_read(filename.c_str(), item.first.offset_, ptr, - total_read_size, item.first.size_); - } - if (read_size > 0) { - total_read_size += read_size; - } - } - existing.first.st_ptr += total_read_size; - struct timespec ts; - timespec_get(&ts, TIME_UTC); - existing.first.st_atim = ts; - existing.first.st_ctim = ts; - mdm->Update(fd, existing.first); - ret = total_read_size; - return ret; -} /** * MPI */ int HERMES_DECL(MPI_Init)(int *argc, char ***argv) { - MAP_OR_FAIL(MPI_Init); - int status = real_MPI_Init_(argc, argv); + auto real_api = Singleton::GetInstance(); + int status = real_api->MPI_Init(argc, argv); if (status == 0) { LOG(INFO) << "MPI Init intercepted." << std::endl; auto mdm = hermes::adapter::Singleton::GetInstance(); @@ -394,11 +57,11 @@ int HERMES_DECL(MPI_Init)(int *argc, char ***argv) { } int HERMES_DECL(MPI_Finalize)(void) { + auto real_api = Singleton::GetInstance(); LOG(INFO) << "MPI Finalize intercepted." << std::endl; auto mdm = hermes::adapter::Singleton::GetInstance(); mdm->FinalizeHermes(); - MAP_OR_FAIL(MPI_Finalize); - int status = real_MPI_Finalize_(); + int status = real_api->MPI_Finalize(); return status; } @@ -406,8 +69,9 @@ int HERMES_DECL(MPI_Finalize)(void) { * POSIX */ int HERMES_DECL(open)(const char *path, int flags, ...) { - int ret; int mode = 0; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (flags & O_CREAT || flags & O_TMPFILE) { va_list arg; va_start(arg, flags); @@ -417,21 +81,21 @@ int HERMES_DECL(open)(const char *path, int flags, ...) { if (hermes::adapter::IsTracked(path)) { LOG(INFO) << "Intercept open for filename: " << path << " and mode: " << flags << " is tracked." << std::endl; - ret = open_internal(path, flags, mode); - } else { - MAP_OR_FAIL(open); - if (flags & O_CREAT || flags & O_TMPFILE) { - ret = real_open_(path, flags, mode); - } else { - ret = real_open_(path, flags); - } + AdapterStat stat; + stat.flags = flags; + stat.st_mode = mode; + return fs_api->Open(stat, path).fd_; } - return (ret); + if (flags & O_CREAT || flags & O_TMPFILE) { + return real_api->open(path, flags, mode); + } + return real_api->open(path, flags); } int HERMES_DECL(open64)(const char *path, int flags, ...) { - int ret; int mode = 0; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (flags & O_CREAT) { va_list arg; va_start(arg, flags); @@ -441,282 +105,178 @@ int HERMES_DECL(open64)(const char *path, int flags, ...) { if (hermes::adapter::IsTracked(path)) { LOG(INFO) << "Intercept open for filename: " << path << " and mode: " << flags << " is tracked." << std::endl; - ret = open_internal(path, flags, mode); - } else { - MAP_OR_FAIL(open64); - if (flags & O_CREAT) { - ret = real_open64_(path, flags, mode); - } else { - ret = real_open64_(path, flags); - } + AdapterStat stat; + stat.flags = flags; + stat.st_mode = mode; + return fs_api->Open(stat, path).fd_; + } + if (flags & O_CREAT) { + return real_api->open64(path, flags, mode); } - return (ret); + return real_api->open64(path, flags); } int HERMES_DECL(__open_2)(const char *path, int oflag) { - int ret; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(path)) { LOG(INFO) << "Intercept __open_2 for filename: " << path << " and mode: " << oflag << " is tracked." << std::endl; - ret = open_internal(path, oflag, 0); - } else { - MAP_OR_FAIL(__open_2); - ret = real___open_2_(path, oflag); + AdapterStat stat; + stat.flags = oflag; + stat.st_mode = 0; + return fs_api->Open(stat, path).fd_; } - return (ret); + return real_api->__open_2(path, oflag); } int HERMES_DECL(creat)(const char *path, mode_t mode) { - int ret; std::string path_str(path); + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(path)) { LOG(INFO) << "Intercept creat for filename: " << path << " and mode: " << mode << " is tracked." << std::endl; - ret = open_internal(path, O_CREAT, mode); - } else { - MAP_OR_FAIL(creat); - ret = real_creat_(path, mode); + AdapterStat stat; + stat.flags = O_CREAT; + stat.st_mode = mode; + return fs_api->Open(stat, path).fd_; } - return (ret); + return real_api->creat(path, mode); } int HERMES_DECL(creat64)(const char *path, mode_t mode) { - int ret; std::string path_str(path); + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(path)) { LOG(INFO) << "Intercept creat64 for filename: " << path << " and mode: " << mode << " is tracked." << std::endl; - ret = open_internal(path, O_CREAT, mode); - } else { - MAP_OR_FAIL(creat64); - ret = real_creat64_(path, mode); + AdapterStat stat; + stat.flags = O_CREAT; + stat.st_mode = mode; + return fs_api->Open(stat, path).fd_; } - return (ret); + return real_api->creat64(path, mode); } ssize_t HERMES_DECL(read)(int fd, void *buf, size_t count) { - size_t ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fd)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fd); - if (existing.second) { - LOG(INFO) << "Intercept read." << std::endl; - ret = read_internal(existing, buf, count, fd); - - } else { - MAP_OR_FAIL(pread); - ret = real_read_(fd, buf, count); - } - } else { - MAP_OR_FAIL(read); - ret = real_read_(fd, buf, count); + LOG(INFO) << "Intercept read." << std::endl; + File f; f.fd_ = fd; fs_api->_InitFile(f); IoStatus io_status; + size_t ret = fs_api->Read(f, stat_exists, buf, count, io_status); + if (stat_exists) return ret; } - return (ret); + return real_api->read(fd, buf, count); } ssize_t HERMES_DECL(write)(int fd, const void *buf, size_t count) { - size_t ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fd)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fd); - if (existing.second) { - LOG(INFO) << "Intercept write." << std::endl; - ret = write_internal(existing.first, buf, count, fd); - } else { - MAP_OR_FAIL(write); - ret = real_write_(fd, buf, count); - } - } else { - MAP_OR_FAIL(write); - ret = real_write_(fd, buf, count); + LOG(INFO) << "Intercept write." << std::endl; + File f; f.fd_ = fd; fs_api->_InitFile(f); IoStatus io_status; + size_t ret = fs_api->Write(f, stat_exists, buf, count, io_status); + if (stat_exists) return ret; } - return (ret); + return real_api->write(fd, buf, count); } ssize_t HERMES_DECL(pread)(int fd, void *buf, size_t count, off_t offset) { - size_t ret = 0; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fd)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fd); - if (existing.second) { - LOG(INFO) << "Intercept pread." << std::endl; - int status = lseek(fd, offset, SEEK_SET); - if (status == 0) { - ret = read_internal(existing, buf, count, fd); - } - } else { - MAP_OR_FAIL(pread); - ret = real_pread_(fd, buf, count, offset); - } - } else { - MAP_OR_FAIL(pread); - ret = real_pread_(fd, buf, count, offset); + LOG(INFO) << "Intercept pread." << std::endl; + File f; f.fd_ = fd; fs_api->_InitFile(f); IoStatus io_status; + size_t ret = fs_api->Read(f, stat_exists, buf, offset, count, io_status); + if (stat_exists) return ret; } - return (ret); + return real_api->pread(fd, buf, count, offset); } ssize_t HERMES_DECL(pwrite)(int fd, const void *buf, size_t count, off_t offset) { - size_t ret = 0; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fd)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fd); - if (existing.second) { - LOG(INFO) << "Intercept pwrite." << std::endl; - int status = lseek(fd, offset, SEEK_SET); - if (status == 0) { - ret = write_internal(existing.first, buf, count, fd); - } - } else { - MAP_OR_FAIL(pwrite); - ret = real_pwrite_(fd, buf, count, offset); - } - } else { - MAP_OR_FAIL(pwrite); - ret = real_pwrite_(fd, buf, count, offset); + File f; f.fd_ = fd; fs_api->_InitFile(f); IoStatus io_status; + LOG(INFO) << "Intercept pwrite." << std::endl; + size_t ret = fs_api->Write(f, stat_exists, buf, offset, count, io_status); + if (stat_exists) return ret; } - return (ret); + return real_api->pwrite(fd, buf, count, offset); } ssize_t HERMES_DECL(pread64)(int fd, void *buf, size_t count, off64_t offset) { - size_t ret = 0; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fd)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fd); - if (existing.second) { - LOG(INFO) << "Intercept pread64." << std::endl; - int status = lseek(fd, offset, SEEK_SET); - if (status == 0) { - ret = read_internal(existing, buf, count, fd); - } - } else { - MAP_OR_FAIL(pread); - ret = real_pread_(fd, buf, count, offset); - } - } else { - MAP_OR_FAIL(pread); - ret = real_pread_(fd, buf, count, offset); + File f; f.fd_ = fd; fs_api->_InitFile(f); IoStatus io_status; + LOG(INFO) << "Intercept pread64." << std::endl; + size_t ret = fs_api->Read(f, stat_exists, buf, offset, count, io_status); + if (stat_exists) return ret; } - return (ret); + return real_api->pread64(fd, buf, count, offset); } ssize_t HERMES_DECL(pwrite64)(int fd, const void *buf, size_t count, off64_t offset) { - size_t ret = 0; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fd)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fd); - if (existing.second) { - LOG(INFO) << "Intercept pwrite." << std::endl; - int status = lseek(fd, offset, SEEK_SET); - if (status == 0) { - ret = write_internal(existing.first, buf, count, fd); - } - } else { - MAP_OR_FAIL(pwrite); - ret = real_pwrite_(fd, buf, count, offset); - } - } else { - MAP_OR_FAIL(pwrite); - ret = real_pwrite_(fd, buf, count, offset); + File f; f.fd_ = fd; fs_api->_InitFile(f); IoStatus io_status; + LOG(INFO) << "Intercept pwrite." << std::endl; + size_t ret = fs_api->Write(f, stat_exists, buf, offset, count, io_status); + if (stat_exists) return ret; } - return (ret); + return real_api->pwrite64(fd, buf, count, offset); } + off_t HERMES_DECL(lseek)(int fd, off_t offset, int whence) { - int ret = -1; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fd)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fd); - if (existing.second) { - LOG(INFO) << "Intercept lseek offset:" << offset << " whence:" << whence - << "." << std::endl; - if (!(existing.first.st_mode & O_APPEND)) { - switch (whence) { - case SEEK_SET: { - existing.first.st_ptr = offset; - break; - } - case SEEK_CUR: { - existing.first.st_ptr += offset; - break; - } - case SEEK_END: { - existing.first.st_ptr = existing.first.st_size + offset; - break; - } - default: { - // TODO(hari): throw not implemented error. - } - } - mdm->Update(fd, existing.first); - ret = existing.first.st_ptr; - } else { - LOG(INFO) - << "File pointer not updating as file was opened in append mode." - << std::endl; - ret = -1; - } - } else { - MAP_OR_FAIL(lseek); - ret = real_lseek_(fd, offset, whence); - } - } else { - MAP_OR_FAIL(lseek); - ret = real_lseek_(fd, offset, whence); + File f; f.fd_ = fd; fs_api->_InitFile(f); + LOG(INFO) << "Intercept lseek offset:" << offset << " whence:" << whence + << "." << std::endl; + return fs_api->Seek(f, stat_exists, + static_cast(whence), offset); } - return (ret); + return real_api->lseek(fd, offset, whence); } + off64_t HERMES_DECL(lseek64)(int fd, off64_t offset, int whence) { - int ret = -1; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fd)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fd); - if (existing.second) { - LOG(INFO) << "Intercept lseek64 offset:" << offset << " whence:" << whence - << "." << std::endl; - if (!(existing.first.st_mode & O_APPEND)) { - switch (whence) { - case SEEK_SET: { - existing.first.st_ptr = offset; - break; - } - case SEEK_CUR: { - existing.first.st_ptr += offset; - break; - } - case SEEK_END: { - existing.first.st_ptr = existing.first.st_size + offset; - break; - } - default: { - // TODO(hari): throw not implemented error. - } - } - mdm->Update(fd, existing.first); - ret = 0; - } else { - LOG(INFO) - << "File pointer not updating as file was opened in append mode." - << std::endl; - ret = -1; - } - } else { - MAP_OR_FAIL(lseek); - ret = real_lseek_(fd, offset, whence); - } - } else { - MAP_OR_FAIL(lseek); - ret = real_lseek_(fd, offset, whence); + File f; f.fd_ = fd; fs_api->_InitFile(f); + LOG(INFO) << "Intercept lseek64 offset:" << offset << " whence:" << whence + << "." << std::endl; + return fs_api->Seek(f, stat_exists, + static_cast(whence), offset); } - return (ret); + return real_api->lseek64(fd, offset, whence); } int HERMES_DECL(__fxstat)(int version, int fd, struct stat *buf) { int result = 0; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fd)) { + File f; f.fd_ = fd; fs_api->_InitFile(f); LOG(INFO) << "Intercepted fstat." << std::endl; auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fd); + auto existing = mdm->Find(f); if (existing.second) { AdapterStat &astat = existing.first; // TODO(chogan): st_dev and st_ino need to be assigned by us, but @@ -737,125 +297,37 @@ int HERMES_DECL(__fxstat)(int version, int fd, struct stat *buf) { } else { result = -1; errno = EBADF; - LOG(ERROR) << "File with descriptor" << fd - << "does not exist in Hermes\n"; + LOG(ERROR) << "File with descriptor " << fd + << " does not exist in Hermes\n"; } } else { - MAP_OR_FAIL(__fxstat); - result = real___fxstat_(version, fd, buf); + result = real_api->__fxstat(version, fd, buf); } - return result; } int HERMES_DECL(fsync)(int fd) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fd)) { + File f; f.fd_ = fd; fs_api->_InitFile(f); LOG(INFO) << "Intercept fsync." << std::endl; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fd); - if (existing.second) { - LOG(INFO) << "File handler is opened by adapter." << std::endl; - hapi::Context ctx; - if (existing.first.ref_count == 1) { - auto persist = INTERCEPTOR_LIST->Persists(fd); - auto filename = existing.first.st_bkid->GetName(); - const auto &blob_names = existing.first.st_blobs; - if (!blob_names.empty() && persist) { - LOG(INFO) << "POSIX fsync Adapter flushes " << blob_names.size() - << " blobs to filename:" << filename << "." << std::endl; - INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); - hapi::VBucket file_vbucket(filename, mdm->GetHermes(), ctx); - auto offset_map = std::unordered_map(); - - for (const auto &blob_name : blob_names) { - auto status = file_vbucket.Link(blob_name, filename, ctx); - if (!status.Failed()) { - auto page_index = std::stol(blob_name) - 1; - offset_map.emplace(blob_name, page_index * kPageSize); - } - } - bool flush_synchronously = true; - hapi::PersistTrait persist_trait(filename, offset_map, - flush_synchronously); - file_vbucket.Attach(&persist_trait, ctx); - file_vbucket.Destroy(ctx); - existing.first.st_blobs.clear(); - INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename); - mdm->Update(fd, existing.first); - } - } else { - LOG(INFO) << "File handler is opened by more than one fopen." - << std::endl; - struct timespec ts; - timespec_get(&ts, TIME_UTC); - existing.first.st_atim = ts; - existing.first.st_ctim = ts; - mdm->Update(fd, existing.first); - } - } + return fs_api->Sync(f, stat_exists); } - - MAP_OR_FAIL(fsync); - ret = real_fsync_(fd); - return (ret); + return real_api->fsync(fd); } int HERMES_DECL(close)(int fd) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fd)) { + File f; f.fd_ = fd; fs_api->_InitFile(f); LOG(INFO) << "Intercept close(" << std::to_string(fd) << ")"; DLOG(INFO) << " -> " << hermes::adapter::GetFilenameFromFD(fd); LOG(INFO) << std::endl; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fd); - if (existing.second) { - LOG(INFO) << "File handler is opened by adapter." << std::endl; - hapi::Context ctx; - if (existing.first.ref_count == 1) { - auto persist = INTERCEPTOR_LIST->Persists(fd); - auto filename = existing.first.st_bkid->GetName(); - mdm->Delete(fd); - const auto &blob_names = existing.first.st_blobs; - if (!blob_names.empty() && persist) { - LOG(INFO) << "POSIX close Adapter flushes " << blob_names.size() - << " blobs to filename:" << filename << "." << std::endl; - INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); - hapi::VBucket file_vbucket(filename, mdm->GetHermes(), ctx); - auto offset_map = std::unordered_map(); - - for (const auto &blob_name : blob_names) { - auto status = file_vbucket.Link(blob_name, filename, ctx); - if (!status.Failed()) { - auto page_index = std::stol(blob_name) - 1; - offset_map.emplace(blob_name, page_index * kPageSize); - } - } - bool flush_synchronously = true; - hapi::PersistTrait persist_trait(filename, offset_map, - flush_synchronously); - file_vbucket.Attach(&persist_trait, ctx); - file_vbucket.Destroy(ctx); - existing.first.st_blobs.clear(); - INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename); - } - existing.first.st_bkid->Destroy(ctx); - mdm->FinalizeHermes(); - } else { - LOG(INFO) << "File handler is opened by more than one fopen." - << std::endl; - existing.first.ref_count--; - struct timespec ts; - timespec_get(&ts, TIME_UTC); - existing.first.st_atim = ts; - existing.first.st_ctim = ts; - mdm->Update(fd, existing.first); - existing.first.st_bkid->Release(ctx); - } - } + return fs_api->Close(f, stat_exists); } - - MAP_OR_FAIL(close); - ret = real_close_(fd); - return (ret); + return real_api->close(fd); } diff --git a/adapter/posix/posix.h b/adapter/posix/posix.h deleted file mode 100644 index 3ec4e5bcb..000000000 --- a/adapter/posix/posix.h +++ /dev/null @@ -1,79 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -// -// Created by hdevarajan on 12/7/20. -// - -#ifndef HERMES_ADAPTER_POSIX_H -#define HERMES_ADAPTER_POSIX_H -/** - * Standard header - */ -#include -#include -#include - -/** - * Dependent library headers - */ -#include - -/** - * Internal headers - */ -#include -#include -#include - -#include "constants.h" -#include "interceptor.h" -#include "singleton.h" -#include "posix/constants.h" -#include "posix/datastructures.h" -#include "posix/mapper/mapper_factory.h" -#include "posix/metadata_manager.h" - -#ifndef O_TMPFILE -#define O_TMPFILE 0 -#endif - -/** - * Function declarations - */ -HERMES_FORWARD_DECL(open, int, (const char *path, int flags, ...)); -HERMES_FORWARD_DECL(open64, int, (const char *path, int flags, ...)); -HERMES_FORWARD_DECL(__open_2, int, (const char *path, int oflag)); -HERMES_FORWARD_DECL(creat, int, (const char *path, mode_t mode)); -HERMES_FORWARD_DECL(creat64, int, (const char *path, mode_t mode)); -HERMES_FORWARD_DECL(read, ssize_t, (int fd, void *buf, size_t count)); -HERMES_FORWARD_DECL(write, ssize_t, (int fd, const void *buf, size_t count)); -HERMES_FORWARD_DECL(pread, ssize_t, - (int fd, void *buf, size_t count, off_t offset)); -HERMES_FORWARD_DECL(pwrite, ssize_t, - (int fd, const void *buf, size_t count, off_t offset)); -HERMES_FORWARD_DECL(pread64, ssize_t, - (int fd, void *buf, size_t count, off64_t offset)); -HERMES_FORWARD_DECL(pwrite64, ssize_t, - (int fd, const void *buf, size_t count, off64_t offset)); -HERMES_FORWARD_DECL(lseek, off_t, (int fd, off_t offset, int whence)); -HERMES_FORWARD_DECL(lseek64, off64_t, (int fd, off64_t offset, int whence)); -HERMES_FORWARD_DECL(__fxstat, int, (int version, int fd, struct stat *buf)); -HERMES_FORWARD_DECL(fsync, int, (int fd)); -HERMES_FORWARD_DECL(fdatasync, int, (int fd)); -HERMES_FORWARD_DECL(close, int, (int fd)); -/** - * MPI function declarations - */ -HERMES_FORWARD_DECL(MPI_Init, int, (int *argc, char ***argv)); -HERMES_FORWARD_DECL(MPI_Finalize, int, (void)); -#endif // HERMES_ADAPTER_POSIX_H diff --git a/adapter/posix/real_api.h b/adapter/posix/real_api.h new file mode 100644 index 000000000..ee8de2f30 --- /dev/null +++ b/adapter/posix/real_api.h @@ -0,0 +1,246 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_ADAPTER_POSIX_H +#define HERMES_ADAPTER_POSIX_H +#include +#include +#include +#include +#include "interceptor.h" +#include "filesystem/filesystem.h" +#include "filesystem/metadata_manager.h" + +namespace hermes::adapter::posix { + +class API { + public: + typedef int(*open_t)(const char * path, int flags, ...); + int(*open)(const char * path, int flags, ...) = nullptr; + typedef int(*open64_t)(const char * path, int flags, ...); + int(*open64)(const char * path, int flags, ...) = nullptr; + typedef int(*__open_2_t)(const char * path, int oflag); + int(*__open_2)(const char * path, int oflag) = nullptr; + typedef int(*creat_t)(const char * path, mode_t mode); + int(*creat)(const char * path, mode_t mode) = nullptr; + typedef int(*creat64_t)(const char * path, mode_t mode); + int(*creat64)(const char * path, mode_t mode) = nullptr; + typedef ssize_t(*read_t)(int fd, void * buf, size_t count); + ssize_t(*read)(int fd, void * buf, size_t count) = nullptr; + typedef ssize_t(*write_t)(int fd, const void * buf, size_t count); + ssize_t(*write)(int fd, const void * buf, size_t count) = nullptr; + typedef ssize_t(*pread_t)(int fd, void * buf, size_t count, off_t offset); + ssize_t(*pread)(int fd, void * buf, size_t count, off_t offset) = nullptr; + typedef ssize_t(*pwrite_t)(int fd, const void * buf, + size_t count, off_t offset); + ssize_t(*pwrite)(int fd, const void * buf, + size_t count, off_t offset) = nullptr; + typedef ssize_t(*pread64_t)(int fd, void * buf, size_t count, off64_t offset); + ssize_t(*pread64)(int fd, void * buf, size_t count, off64_t offset) = nullptr; + typedef ssize_t(*pwrite64_t)(int fd, const void * buf, + size_t count, off64_t offset); + ssize_t(*pwrite64)(int fd, const void * buf, + size_t count, off64_t offset) = nullptr; + typedef off_t(*lseek_t)(int fd, off_t offset, int whence); + off_t(*lseek)(int fd, off_t offset, int whence) = nullptr; + typedef off64_t(*lseek64_t)(int fd, off64_t offset, int whence); + off64_t(*lseek64)(int fd, off64_t offset, int whence) = nullptr; + typedef int(*__fxstat_t)(int version, int fd, struct stat * buf); + int(*__fxstat)(int version, int fd, struct stat * buf) = nullptr; + typedef int(*fsync_t)(int fd); + int(*fsync)(int fd) = nullptr; + typedef int(*fdatasync_t)(int fd); + int(*fdatasync)(int fd) = nullptr; + typedef int(*close_t)(int fd); + int(*close)(int fd) = nullptr; + typedef int(*MPI_Init_t)(int * argc, char *** argv); + int(*MPI_Init)(int * argc, char *** argv) = nullptr; + typedef int(*MPI_Finalize_t)(); + int(*MPI_Finalize)() = nullptr; + API() { + void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, "posix_intercepted"); + if (is_intercepted) { + open = (open_t)dlsym(RTLD_NEXT, "open"); + } else { + open = (open_t)dlsym(RTLD_DEFAULT, "open"); + } + if (open == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "open" << std::endl; + } + if (is_intercepted) { + open64 = (open64_t)dlsym(RTLD_NEXT, "open64"); + } else { + open64 = (open64_t)dlsym(RTLD_DEFAULT, "open64"); + } + if (open64 == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "open64" << std::endl; + } + if (is_intercepted) { + __open_2 = (__open_2_t)dlsym(RTLD_NEXT, "__open_2"); + } else { + __open_2 = (__open_2_t)dlsym(RTLD_DEFAULT, "__open_2"); + } + if (__open_2 == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "__open_2" << std::endl; + } + if (is_intercepted) { + creat = (creat_t)dlsym(RTLD_NEXT, "creat"); + } else { + creat = (creat_t)dlsym(RTLD_DEFAULT, "creat"); + } + if (creat == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "creat" << std::endl; + } + if (is_intercepted) { + creat64 = (creat64_t)dlsym(RTLD_NEXT, "creat64"); + } else { + creat64 = (creat64_t)dlsym(RTLD_DEFAULT, "creat64"); + } + if (creat64 == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "creat64" << std::endl; + } + if (is_intercepted) { + read = (read_t)dlsym(RTLD_NEXT, "read"); + } else { + read = (read_t)dlsym(RTLD_DEFAULT, "read"); + } + if (read == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "read" << std::endl; + } + if (is_intercepted) { + write = (write_t)dlsym(RTLD_NEXT, "write"); + } else { + write = (write_t)dlsym(RTLD_DEFAULT, "write"); + } + if (write == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "write" << std::endl; + } + if (is_intercepted) { + pread = (pread_t)dlsym(RTLD_NEXT, "pread"); + } else { + pread = (pread_t)dlsym(RTLD_DEFAULT, "pread"); + } + if (pread == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "pread" << std::endl; + } + if (is_intercepted) { + pwrite = (pwrite_t)dlsym(RTLD_NEXT, "pwrite"); + } else { + pwrite = (pwrite_t)dlsym(RTLD_DEFAULT, "pwrite"); + } + if (pwrite == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "pwrite" << std::endl; + } + if (is_intercepted) { + pread64 = (pread64_t)dlsym(RTLD_NEXT, "pread64"); + } else { + pread64 = (pread64_t)dlsym(RTLD_DEFAULT, "pread64"); + } + if (pread64 == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "pread64" << std::endl; + } + if (is_intercepted) { + pwrite64 = (pwrite64_t)dlsym(RTLD_NEXT, "pwrite64"); + } else { + pwrite64 = (pwrite64_t)dlsym(RTLD_DEFAULT, "pwrite64"); + } + if (pwrite64 == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "pwrite64" << std::endl; + } + if (is_intercepted) { + lseek = (lseek_t)dlsym(RTLD_NEXT, "lseek"); + } else { + lseek = (lseek_t)dlsym(RTLD_DEFAULT, "lseek"); + } + if (lseek == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "lseek" << std::endl; + } + if (is_intercepted) { + lseek64 = (lseek64_t)dlsym(RTLD_NEXT, "lseek64"); + } else { + lseek64 = (lseek64_t)dlsym(RTLD_DEFAULT, "lseek64"); + } + if (lseek64 == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "lseek64" << std::endl; + } + if (is_intercepted) { + __fxstat = (__fxstat_t)dlsym(RTLD_NEXT, "__fxstat"); + } else { + __fxstat = (__fxstat_t)dlsym(RTLD_DEFAULT, "__fxstat"); + } + if (__fxstat == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "__fxstat" << std::endl; + } + if (is_intercepted) { + fsync = (fsync_t)dlsym(RTLD_NEXT, "fsync"); + } else { + fsync = (fsync_t)dlsym(RTLD_DEFAULT, "fsync"); + } + if (fsync == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fsync" << std::endl; + } + if (is_intercepted) { + fdatasync = (fdatasync_t)dlsym(RTLD_NEXT, "fdatasync"); + } else { + fdatasync = (fdatasync_t)dlsym(RTLD_DEFAULT, "fdatasync"); + } + if (fdatasync == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fdatasync" << std::endl; + } + if (is_intercepted) { + close = (close_t)dlsym(RTLD_NEXT, "close"); + } else { + close = (close_t)dlsym(RTLD_DEFAULT, "close"); + } + if (close == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "close" << std::endl; + } + if (is_intercepted) { + MPI_Init = (MPI_Init_t)dlsym(RTLD_NEXT, "MPI_Init"); + } else { + MPI_Init = (MPI_Init_t)dlsym(RTLD_DEFAULT, "MPI_Init"); + } + if (MPI_Init == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_Init" << std::endl; + } + if (is_intercepted) { + MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_NEXT, "MPI_Finalize"); + } else { + MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_DEFAULT, "MPI_Finalize"); + } + if (MPI_Finalize == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_Finalize" << std::endl; + } + } +}; +} // namespace hermes::adapter::posix + +#endif // HERMES_ADAPTER_POSIX_H diff --git a/adapter/singleton.h b/adapter/singleton.h index e3dadbf7a..9ab2a1959 100644 --- a/adapter/singleton.h +++ b/adapter/singleton.h @@ -31,10 +31,10 @@ class Singleton { * @return instance of T */ template - static std::shared_ptr GetInstance(Args... args) { + static T* GetInstance(Args... args) { if (instance == nullptr) - instance = std::make_shared(std::forward(args)...); - return instance; + instance = std::make_unique(std::forward(args)...); + return instance.get(); } /** @@ -52,11 +52,11 @@ class Singleton { /** * static instance. */ - static std::shared_ptr instance; + static std::unique_ptr instance; Singleton() {} /* hidden default constructor. */ }; template -std::shared_ptr Singleton::instance = nullptr; +std::unique_ptr Singleton::instance = nullptr; } // namespace hermes::adapter #endif // HERMES_ADAPTER_SINGLETON_H diff --git a/adapter/stdio/CMakeLists.txt b/adapter/stdio/CMakeLists.txt index 2598600ed..fb34ba472 100644 --- a/adapter/stdio/CMakeLists.txt +++ b/adapter/stdio/CMakeLists.txt @@ -1,4 +1,5 @@ project(StdioAdapter VERSION ${HERMES_PACKAGE_VERSION}) +include_directories(${HERMES_ADAPTER_DIR} ${CMAKE_CURRENT_SOURCE_DIR}) # STDIO src code. We only include stdio.cc as it includes other cc to reduce compilation time. set(STDIO_ADAPTER_SRC stdio.cc) @@ -6,25 +7,31 @@ set(STDIO_ADAPTER_SRC stdio.cc) set(HERMES_STDIO_ADAPTER_DIR ${HERMES_ADAPTER_DIR}/stdio) # Only stdio.h is the public adapter. -set(STDIO_ADAPTER_PUBLIC_HEADER ${HERMES_STDIO_ADAPTER_DIR}/stdio.h) -# Private headers -set(STDIO_ADAPTER_PRIVATE_HEADER ${CMAKE_CURRENT_SOURCE_DIR}/metadata_manager.h - ${CMAKE_CURRENT_SOURCE_DIR}/mapper/mapper_factory.h - ${CMAKE_CURRENT_SOURCE_DIR}/mapper/abstract_mapper.h - ${CMAKE_CURRENT_SOURCE_DIR}/mapper/balanced_mapper.h - ${CMAKE_CURRENT_SOURCE_DIR}/datastructures.h - ${CMAKE_CURRENT_SOURCE_DIR}/enumerations.h - ${CMAKE_CURRENT_SOURCE_DIR}/constants.h) +set(STDIO_ADAPTER_PUBLIC_HEADER + ${HERMES_STDIO_ADAPTER_DIR}/real_api.h + ${HERMES_STDIO_ADAPTER_DIR}/fs_api.h) # Add library hermes_stdio -add_library(hermes_stdio SHARED ${STDIO_ADAPTER_PRIVATE_HEADER} ${STDIO_ADAPTER_PUBLIC_HEADER} ${STDIO_ADAPTER_SRC}) -target_include_directories(hermes_stdio PRIVATE ${HERMES_ADAPTER_DIR}) -add_dependencies(hermes_stdio hermes) -target_link_libraries(hermes_stdio hermes MPI::MPI_CXX glog::glog stdc++fs) +add_library(hermes_stdio_backend ${CMAKE_CURRENT_SOURCE_DIR}/fs_api.cc) +add_dependencies(hermes_stdio_backend hermes) +target_link_libraries(hermes_stdio_backend hermes MPI::MPI_CXX glog::glog stdc++fs dl) + +add_library(hermes_stdio SHARED ${STDIO_ADAPTER_PUBLIC_HEADER} ${STDIO_ADAPTER_SRC}) +add_dependencies(hermes_stdio hermes_stdio_backend) +target_link_libraries(hermes_stdio hermes_stdio_backend) #----------------------------------------------------------------------------- # Add Target(s) to CMake Install #----------------------------------------------------------------------------- +install( + TARGETS + hermes_stdio_backend + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} +) install( TARGETS hermes_stdio diff --git a/adapter/stdio/constants.h b/adapter/stdio/constants.h deleted file mode 100644 index 7ad19bd9c..000000000 --- a/adapter/stdio/constants.h +++ /dev/null @@ -1,35 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_STDIO_COMMON_CONSTANTS_H -#define HERMES_STDIO_COMMON_CONSTANTS_H - -#include - -#include "stdio/enumerations.h" - -/** - * Constants file for STDIO adapter. - */ -using hermes::adapter::stdio::MapperType; - -/** - * Which mapper to be used by STDIO adapter. - */ -const MapperType kMapperType = MapperType::BALANCED; - -/** - * String delimiter - */ -const char kStringDelimiter = '#'; - -#endif // HERMES_STDIO_COMMON_CONSTANTS_H diff --git a/adapter/stdio/datastructures.h b/adapter/stdio/datastructures.h deleted file mode 100644 index c25707422..000000000 --- a/adapter/stdio/datastructures.h +++ /dev/null @@ -1,244 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_STDIO_ADAPTER_DATASTRUCTURES_H -#define HERMES_STDIO_ADAPTER_DATASTRUCTURES_H - -/** - * Standard header - */ -#include -#include -#include - -/** - * Dependent library header - */ - -/** - * Internal header - */ -#include -#include -#include -#include -#include - -/** - * Namespace simplification. - */ -namespace hapi = hermes::api; - -namespace hermes::adapter::stdio { - -/** - * FileID structure used as an identifier in STDIO adapter. - */ -struct FileID { - /** - * attributes - */ - dev_t dev_id_; // device id to place the file. - ino_t inode_num_; // inode number refering to the file. - /** - * Constructor - */ - FileID() : dev_id_(), inode_num_() {} /* default constructor */ - FileID(dev_t dev_id, ino_t inode_num) - : dev_id_(dev_id), - inode_num_(inode_num) {} /* parameterized constructor */ - FileID(const FileID &other) - : dev_id_(other.dev_id_), - inode_num_(other.inode_num_) {} /* copy constructor*/ - FileID(FileID &&other) - : dev_id_(other.dev_id_), - inode_num_(other.inode_num_) {} /* move constructor*/ - - /** - * Operators defined - */ - /* Assignment operator. */ - FileID &operator=(const FileID &other) { - dev_id_ = other.dev_id_; - inode_num_ = other.inode_num_; - return *this; - } - - /* Equal operator. */ - bool operator==(const FileID &o) const { - return dev_id_ == o.dev_id_ && inode_num_ == o.inode_num_; - } -}; - -/** - * Structure STDIO adapter uses to define a file state. - */ -struct FileStruct { - /** - * attributes - */ - FileID file_id_; // fileID to identify a file uniquely. - size_t offset_; // file pointer within the file. - size_t size_; // size of data refered in file. - /** - * Constructor - */ - FileStruct() : file_id_(), offset_(0), size_(0) {} /* default constructor */ - FileStruct(FileID file_id, size_t offset, size_t size) - : file_id_(file_id), - offset_(offset), - size_(size) {} /* parameterized constructor */ - FileStruct(const FileStruct &other) - : file_id_(other.file_id_), - offset_(other.offset_), - size_(other.size_) {} /* copy constructor*/ - FileStruct(FileStruct &&other) - : file_id_(other.file_id_), - offset_(other.offset_), - size_(other.size_) {} /* move constructor*/ - /** - * Operators defined - */ - /* Assignment operator. */ - FileStruct &operator=(const FileStruct &other) { - file_id_ = other.file_id_; - offset_ = other.offset_; - size_ = other.size_; - return *this; - } -}; - -/** - * Structure STDIO adapter uses to define Hermes blob. - */ -struct HermesStruct { - /** - * attributes - */ - std::string blob_name_; - size_t offset_; - size_t size_; - /** - * Constructor - */ - HermesStruct() - : blob_name_(), offset_(0), size_(0) {} /* default constructor */ - HermesStruct(const HermesStruct &other) - : blob_name_(other.blob_name_), - offset_(other.offset_), - size_(other.size_) {} /* copy constructor*/ - HermesStruct(HermesStruct &&other) - : blob_name_(other.blob_name_), - offset_(other.offset_), - size_(other.size_) {} /* move constructor*/ - /** - * Operators defined - */ - /* Assignment operator. */ - HermesStruct &operator=(const HermesStruct &other) { - blob_name_ = other.blob_name_; - offset_ = other.offset_; - size_ = other.size_; - return *this; - } -}; - -/** - * Stat which defines File within STDIO Adapter. - */ -struct AdapterStat { - /** - * attributes - */ - std::shared_ptr st_bkid; /* bucket associated with the file */ - /** VBucket for persisting data asynchronously. */ - std::shared_ptr st_vbkt; - /** Used for async flushing. */ - std::shared_ptr st_persist; - std::set - st_blobs; /* Blobs access in the bucket */ - i32 ref_count; /* # of time process opens a file */ - mode_t st_mode; /* protection */ - uid_t st_uid; /* user ID of owner */ - gid_t st_gid; /* group ID of owner */ - off_t st_size; /* total size, in bytes */ - off_t st_ptr; /* Current ptr of FILE */ - blksize_t st_blksize; /* blocksize for blob within bucket */ - timespec st_atim; /* time of last access */ - timespec st_mtim; /* time of last modification */ - timespec st_ctim; /* time of last status change */ - /** - * Constructor - */ - AdapterStat() - : st_bkid(), - st_vbkt(), - st_persist(), - st_blobs(CompareBlobs), - ref_count(), - st_mode(), - st_uid(), - st_gid(), - st_size(0), - st_ptr(0), - st_blksize(4096), - st_atim(), - st_mtim(), - st_ctim() {} /* default constructor */ - explicit AdapterStat(const struct stat &st) - : st_bkid(), - st_vbkt(), - st_persist(), - st_blobs(CompareBlobs), - ref_count(1), - st_mode(st.st_mode), - st_uid(st.st_uid), - st_gid(st.st_gid), - st_size(st.st_size), - st_ptr(0), - st_blksize(st.st_blksize), - st_atim(st.st_atim), - st_mtim(st.st_mtim), - st_ctim(st.st_ctim) {} /* parameterized constructor */ - - /** - * Comparator for comparing two blobs. - */ - static bool CompareBlobs(const std::string &a, const std::string &b) { - return std::stol(a) < std::stol(b); - } -}; - -} // namespace hermes::adapter::stdio - -/** - * Define hash functions for STDIO Adapter. - */ -namespace std { - -/** - * hash for FileID. - */ -template <> -struct hash { - std::size_t operator()(const hermes::adapter::stdio::FileID &key) const { - std::size_t result = 0; - std::size_t h1 = std::hash{}(key.dev_id_); - std::size_t h2 = std::hash{}(key.inode_num_); - result = h1 ^ (h2 << 1); - - return result; - } -}; -} // namespace std - -#endif // HERMES_STDIO_ADAPTER_DATASTRUCTURES_H diff --git a/adapter/stdio/enumerations.h b/adapter/stdio/enumerations.h deleted file mode 100644 index 14110bf3d..000000000 --- a/adapter/stdio/enumerations.h +++ /dev/null @@ -1,25 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_STDIO_COMMON_ENUMERATIONS_H -#define HERMES_STDIO_COMMON_ENUMERATIONS_H -/** - * Enumeration for STDIO adapter. - */ -namespace hermes::adapter::stdio { -/** - * Define different types of mappers supported by STDIO Adapter. - * Also define its construction in the MapperFactory. - */ -enum MapperType { BALANCED = 0 /* Balanced Mapping */ }; -} // namespace hermes::adapter::stdio -#endif // HERMES_STDIO_COMMON_ENUMERATIONS_H diff --git a/adapter/stdio/fs_api.cc b/adapter/stdio/fs_api.cc new file mode 100644 index 000000000..e2ddd640d --- /dev/null +++ b/adapter/stdio/fs_api.cc @@ -0,0 +1,109 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include "real_api.h" +#include "fs_api.h" +#include + +namespace hermes::adapter::stdio { + +File StdioFS::_RealOpen(AdapterStat &stat, const std::string &path) { + File f; + f.fh_ = real_api->fopen(path.c_str(), stat.mode_str.c_str()); + if (f.fh_ == nullptr) { + f.status_ = false; + } + _InitFile(f); + return f; +} + +void StdioFS::_InitFile(File &f) { + struct stat st; + if (f.fh_ == nullptr) { + f.fd_ = -1; + return; + } + f.fd_ = fileno(f.fh_); + posix_api->__fxstat(_STAT_VER, f.fd_, &st); + f.st_dev = st.st_dev; + f.st_ino = st.st_ino; +} + +void StdioFS::_OpenInitStats(File &f, AdapterStat &stat, bool bucket_exists) { + (void) bucket_exists; + struct stat st; + posix_api->__fxstat(_STAT_VER, f.fd_, &st); + stat.st_mode = st.st_mode; + stat.st_uid = st.st_uid; + stat.st_gid = st.st_gid; + stat.st_size = st.st_size; + stat.st_blksize = st.st_blksize; + stat.st_atim = st.st_atim; + stat.st_mtim = st.st_mtim; + stat.st_ctim = st.st_ctim; + /*if (bucket_exists) { + stat.st_size = stat.st_bkid->GetTotalBlobSize(); + LOG(INFO) << "Since bucket exists, should reset its size to: " << stat.st_size + << std::endl; + }*/ + if (stat.mode_str.find('a') != std::string::npos) { + stat.st_ptr = stat.st_size; + stat.is_append = true; + } +} + +size_t StdioFS::_RealWrite(const std::string &filename, off_t offset, + size_t size, const u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) { + (void) opts; (void) io_status; + LOG(INFO) << "Writing to file: " << filename + << " offset: " << offset + << " size:" << size << "." + << " file_size:" << stdfs::file_size(filename) << std::endl; + FILE *fh = real_api->fopen(filename.c_str(), "r+"); + if (fh == nullptr) { return 0; } + real_api->fseek(fh, offset, SEEK_SET); + flock(fileno(fh), LOCK_EX); + size_t write_size = real_api->fwrite(data_ptr, sizeof(char), size, fh); + flock(fileno(fh), LOCK_UN); + real_api->fclose(fh); + return write_size; +} + +size_t StdioFS::_RealRead(const std::string &filename, off_t offset, + size_t size, u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) { + (void) opts; (void) io_status; + LOG(INFO) << "Read called for filename from destination: " << filename + << " on offset: " << offset + << " and size: " << size << "." + << " file_size:" << stdfs::file_size(filename) << std::endl; + FILE *fh = real_api->fopen(filename.c_str(), "r"); + if (fh == nullptr) { return 0; } + real_api->fseek(fh, offset, SEEK_SET); + flock(fileno(fh), LOCK_SH); + size_t read_size = real_api->fread(data_ptr, sizeof(char), size, fh); + flock(fileno(fh), LOCK_UN); + real_api->fclose(fh); + return read_size; +} + +int StdioFS::_RealSync(File &f) { + return real_api->fflush(f.fh_); +} + +int StdioFS::_RealClose(File &f) { + return real_api->fclose(f.fh_); +} + +} // namespace hermes::adapter::stdio diff --git a/adapter/stdio/fs_api.h b/adapter/stdio/fs_api.h new file mode 100644 index 000000000..c3550342e --- /dev/null +++ b/adapter/stdio/fs_api.h @@ -0,0 +1,61 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_ADAPTER_STDIO_NATIVE_H_ +#define HERMES_ADAPTER_STDIO_NATIVE_H_ + +#include +#include "filesystem/filesystem.h" +#include "filesystem/filesystem.cc" +#include "filesystem/metadata_manager.h" +#include "filesystem/metadata_manager.cc" +#include "posix/real_api.h" +#include "real_api.h" + +using hermes::adapter::fs::AdapterStat; +using hermes::adapter::fs::File; +using hermes::adapter::Singleton; +using hermes::adapter::stdio::API; +using hermes::adapter::fs::IoOptions; +using hermes::adapter::fs::IoStatus; + +namespace hermes::adapter::stdio { + +class StdioFS : public hermes::adapter::fs::Filesystem { + private: + API* real_api; + hermes::adapter::posix::API* posix_api; + public: + StdioFS() { + real_api = Singleton::GetInstance(); + posix_api = Singleton::GetInstance(); + } + ~StdioFS() = default; + + void _InitFile(File &f) override; + + private: + void _OpenInitStats(File &f, AdapterStat &stat, bool bucket_exists) override; + File _RealOpen(AdapterStat &stat, const std::string &path) override; + size_t _RealWrite(const std::string &filename, off_t offset, size_t size, + const u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) override; + size_t _RealRead(const std::string &filename, off_t offset, size_t size, + u8 *data_ptr, + IoStatus &io_status, IoOptions &opts) override; + int _RealSync(File &f) override; + int _RealClose(File &f) override; +}; + +} // namespace hermes::adapter::stdio + +#endif // HERMES_ADAPTER_STDIO_NATIVE_H_ diff --git a/adapter/stdio/mapper/abstract_mapper.h b/adapter/stdio/mapper/abstract_mapper.h deleted file mode 100644 index b626caa1b..000000000 --- a/adapter/stdio/mapper/abstract_mapper.h +++ /dev/null @@ -1,45 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -// -// Created by manihariharan on 12/23/20. -// - -#ifndef HERMES_ABSTRACT_ADAPTER_H -#define HERMES_ABSTRACT_ADAPTER_H - -#include "stdio/datastructures.h" - -/** - * Typedef to simplify the return types - */ -typedef std::vector> - MapperReturnType; - -namespace hermes::adapter::stdio { -/** - * Interface to define a mapper. - */ -class AbstractMapper { - public: - /** - * This method maps the current Operation to Hermes data structures. - * - * @param file_op, FileStruct, operations for which we are mapping. - * @return a map of FileStruct to Hermes Struct - */ - virtual MapperReturnType map(const FileStruct& file_op) = 0; -}; -} // namespace hermes::adapter::stdio - -#endif // HERMES_ABSTRACT_ADAPTER_H diff --git a/adapter/stdio/mapper/balanced_mapper.cc b/adapter/stdio/mapper/balanced_mapper.cc deleted file mode 100644 index 40a9a3663..000000000 --- a/adapter/stdio/mapper/balanced_mapper.cc +++ /dev/null @@ -1,46 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "balanced_mapper.h" - -/** - * Namespace declaration for cleaner code. - */ -using hermes::adapter::stdio::BalancedMapper; -using hermes::adapter::stdio::FileStruct; -using hermes::adapter::stdio::HermesStruct; - -MapperReturnType BalancedMapper::map(const FileStruct& file_op) { - VLOG(1) << "Mapping File with offset:" << file_op.offset_ - << " and size:" << file_op.size_ << "." << std::endl; - - auto mapper_return = MapperReturnType(); - size_t size_mapped = 0; - while (file_op.size_ > size_mapped) { - FileStruct file; - file.file_id_ = file_op.file_id_; - HermesStruct hermes; - file.offset_ = file_op.offset_ + size_mapped; - size_t page_index = file.offset_ / kPageSize; - hermes.offset_ = file.offset_ % kPageSize; - auto left_size_page = kPageSize - hermes.offset_; - hermes.size_ = left_size_page < file_op.size_ - size_mapped - ? left_size_page - : file_op.size_ - size_mapped; - - file.size_ = hermes.size_; - hermes.blob_name_ = std::to_string(page_index + 1); - mapper_return.emplace_back(file, hermes); - size_mapped += hermes.size_; - } - return mapper_return; -} diff --git a/adapter/stdio/mapper/mapper_factory.h b/adapter/stdio/mapper/mapper_factory.h deleted file mode 100644 index a125c6982..000000000 --- a/adapter/stdio/mapper/mapper_factory.h +++ /dev/null @@ -1,55 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_ADAPTER_FACTORY_H -#define HERMES_ADAPTER_FACTORY_H - -/** - * Standard header - */ - -/** - * Dependent library header - */ - -/** - * Internal header - */ -#include "singleton.h" -#include "stdio/enumerations.h" - -#include "abstract_mapper.h" -#include "balanced_mapper.h" - -namespace hermes::adapter::stdio { -class MapperFactory { - public: - /** - * Return the instance of mapper given a type. Uses factory pattern. - * - * @param type, MapperType, type of mapper to be used by the STDIO adapter. - * @return Instance of mapper given a type. - */ - std::shared_ptr Get(const MapperType &type) { - switch (type) { - case MapperType::BALANCED: { - return hermes::adapter::Singleton::GetInstance(); - } - default: { - // TODO(hari): @errorhandling Mapper not implemented - } - } - return NULL; - } -}; -} // namespace hermes::adapter::stdio -#endif // HERMES_ADAPTER_FACTORY_H diff --git a/adapter/stdio/metadata_manager.cc b/adapter/stdio/metadata_manager.cc deleted file mode 100644 index 7bd8ba602..000000000 --- a/adapter/stdio/metadata_manager.cc +++ /dev/null @@ -1,72 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include "metadata_manager.h" - -/** - * Namespace declarations for cleaner code. - */ -using hermes::adapter::stdio::AdapterStat; -using hermes::adapter::stdio::FileID; -using hermes::adapter::stdio::MetadataManager; - -bool MetadataManager::Create(FILE *fh, const AdapterStat &stat) { - VLOG(1) << "Create metadata for file handler." << std::endl; - auto ret = metadata.emplace(Convert(fh), stat); - return ret.second; -} - -bool MetadataManager::Update(FILE *fh, const AdapterStat &stat) { - auto fileId = Convert(fh); - auto iter = metadata.find(fileId); - if (iter != metadata.end()) { - metadata.erase(iter); - auto ret = metadata.emplace(fileId, stat); - return ret.second; - } else { - return false; - } -} - -std::pair MetadataManager::Find(FILE *fh) { - auto fileId = Convert(fh); - typedef std::pair MetadataReturn; - auto iter = metadata.find(fileId); - if (iter == metadata.end()) - return MetadataReturn(AdapterStat(), false); - else - return MetadataReturn(iter->second, true); -} - -FileID MetadataManager::Convert(FILE *fh) { - struct stat st; - int fd = fileno(fh); - int status = fstat(fd, &st); - if (status == 0) { - return FileID(st.st_dev, st.st_ino); - } else { - // TODO(hari) @errorhandling fstat failed invalid fh. - return FileID(); - } -} - -bool MetadataManager::Delete(FILE *fh) { - VLOG(1) << "Delete metadata for file handler." << std::endl; - auto fileId = Convert(fh); - auto iter = metadata.find(fileId); - if (iter != metadata.end()) { - metadata.erase(iter); - return true; - } else { - return false; - } -} diff --git a/adapter/stdio/metadata_manager.h b/adapter/stdio/metadata_manager.h deleted file mode 100644 index e9d556d11..000000000 --- a/adapter/stdio/metadata_manager.h +++ /dev/null @@ -1,177 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_ADAPTER_METADATA_MANAGER_H -#define HERMES_ADAPTER_METADATA_MANAGER_H - -#include -#include -#include - -#include - -#include "constants.h" -#include "enumerations.h" -#include "interceptor.h" -#include "stdio/constants.h" -#include "stdio/datastructures.h" - -namespace hermes::adapter::stdio { - -FlushingMode global_flushing_mode; - -/** - * Metadata manager for STDIO adapter - */ -class MetadataManager { - private: - /** - * Private members - */ - /** - * Maintain a local metadata FileID structure mapped to Adapter Stats. - */ - std::unordered_map metadata; - /** - * hermes attribute to initialize Hermes - */ - std::shared_ptr hermes; - /** - * references of how many times hermes was tried to initialize. - */ - std::atomic ref; - /** - * MPI attributes - */ - bool is_mpi; - int rank; - int comm_size; - - public: - /** - * Constructor - */ - MetadataManager() - : metadata(), - ref(0), - is_mpi(false), - rank(0), - comm_size(1) {} - /** - * Get the instance of hermes. - */ - std::shared_ptr& GetHermes() { return hermes; } - - - /** - * Initialize hermes. Get the kHermesConf from environment else get_env - * returns NULL which is handled internally by hermes. Initialize hermes in - * daemon mode. Keep a reference of how many times Initialize is called. - * Within the adapter, Initialize is called from fopen. - */ - void InitializeHermes(bool is_mpi = false) { - if (ref == 0) { - this->is_mpi = is_mpi; - char* async_flush_mode = getenv(kHermesAsyncFlush); - - if (async_flush_mode && async_flush_mode[0] == '1') { - global_flushing_mode = FlushingMode::kAsynchronous; - } else { - global_flushing_mode = FlushingMode::kSynchronous; - } - - char* hermes_config = getenv(kHermesConf); - - LOG(INFO) << "InitializeHermes (stdio): " << this->is_mpi << std::endl; - if (this->is_mpi) { - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &comm_size); - // TODO(chogan): Need a better way to distinguish between client and - // daemon. https://github.com/HDFGroup/hermes/issues/206 - if (comm_size > 1) { - hermes = hermes::InitHermesClient(hermes_config); - } else { - this->is_mpi = false; - hermes = hermes::InitHermesDaemon(hermes_config); - } - } else { - hermes = hermes::InitHermesDaemon(hermes_config); - } - INTERCEPTOR_LIST->SetupAdapterMode(); - } - ref++; - } - /** - * Finalize hermes and close rpc if reference is equal to one. Else just - * decrement the ref counter. - */ - void FinalizeHermes() { - if (ref == 1) { - if (this->is_mpi) { - MPI_Barrier(MPI_COMM_WORLD); - char *stop_daemon = getenv(kStopDaemon); - bool shutdown_daemon = true; - - if (stop_daemon && stop_daemon[0] == '0') { - shutdown_daemon = false; - } - - hermes->FinalizeClient(shutdown_daemon); - } else { - hermes->Finalize(true); - } - } - ref--; - } - /** - * Convert file handler to FileID using the stat. - */ - FileID Convert(FILE* fh); - - /** - * Create a metadata entry for STDIO adapter for a given file handler. - * @param fh, FILE*, original file handler of the file on the destination - * filesystem. - * @param stat, AdapterStat, STDIO Adapter version of Stat data structure. - * @return true, if operation was successful. - * false, if operation was unsuccessful. - */ - bool Create(FILE* fh, const AdapterStat& stat); - - /** - * Update existing metadata entry for STDIO adapter for a given file handler. - * @param fh, FILE*, original file handler of the file on the destination. - * @param stat, AdapterStat, STDIO Adapter version of Stat data structure. - * @return true, if operation was successful. - * false, if operation was unsuccessful or entry doesn't exist. - */ - bool Update(FILE* fh, const AdapterStat& stat); - - /** - * Delete existing metadata entry for STDIO adapter for a given file handler. - * @param fh, FILE*, original file handler of the file on the destination. - * @return true, if operation was successful. - * false, if operation was unsuccessful. - */ - bool Delete(FILE* fh); - - /** - * Find existing metadata entry for STDIO adapter for a given file handler. - * @param fh, FILE*, original file handler of the file on the destination. - * @return The metadata entry if exist. - * The bool in pair indicated whether metadata entry exists. - */ - std::pair Find(FILE* fh); -}; -} // namespace hermes::adapter::stdio - -#endif // HERMES_ADAPTER_METADATA_MANAGER_H diff --git a/adapter/stdio/real_api.h b/adapter/stdio/real_api.h new file mode 100644 index 000000000..c4d702452 --- /dev/null +++ b/adapter/stdio/real_api.h @@ -0,0 +1,352 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Distributed under BSD 3-Clause license. * + * Copyright by The HDF Group. * + * Copyright by the Illinois Institute of Technology. * + * All rights reserved. * + * * + * This file is part of Hermes. The full Hermes copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the top directory. If you do not * + * have access to the file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_ADAPTER_STDIO_H +#define HERMES_ADAPTER_STDIO_H +#include +#include +#include +#include +#include "interceptor.h" +#include "filesystem/filesystem.h" +#include + +namespace hermes::adapter::stdio { + +class API { + public: + typedef int (*MPI_Init_t)(int * argc, char *** argv); + int (*MPI_Init)(int * argc, char *** argv) = nullptr; + typedef int (*MPI_Finalize_t)( void); + int (*MPI_Finalize)( void) = nullptr; + typedef FILE * (*fopen_t)(const char * path, const char * mode); + FILE * (*fopen)(const char * path, const char * mode) = nullptr; + typedef FILE * (*fopen64_t)(const char * path, const char * mode); + FILE * (*fopen64)(const char * path, const char * mode) = nullptr; + typedef FILE * (*fdopen_t)(int fd, const char * mode); + FILE * (*fdopen)(int fd, const char * mode) = nullptr; + typedef FILE * (*freopen_t)(const char * path, + const char * mode, FILE * stream); + FILE * (*freopen)(const char * path, + const char * mode, FILE * stream) = nullptr; + typedef FILE * (*freopen64_t)(const char * path, + const char * mode, FILE * stream); + FILE * (*freopen64)(const char * path, + const char * mode, FILE * stream) = nullptr; + typedef int (*fflush_t)(FILE * fp); + int (*fflush)(FILE * fp) = nullptr; + typedef int (*fclose_t)(FILE * fp); + int (*fclose)(FILE * fp) = nullptr; + typedef size_t (*fwrite_t)(const void * ptr, size_t size, + size_t nmemb, FILE * fp); + size_t (*fwrite)(const void * ptr, size_t size, + size_t nmemb, FILE * fp) = nullptr; + typedef int (*fputc_t)(int c, FILE * fp); + int (*fputc)(int c, FILE * fp) = nullptr; + typedef int (*fgetpos_t)(FILE * fp, fpos_t * pos); + int (*fgetpos)(FILE * fp, fpos_t * pos) = nullptr; + typedef int (*fgetpos64_t)(FILE * fp, fpos64_t * pos); + int (*fgetpos64)(FILE * fp, fpos64_t * pos) = nullptr; + typedef int (*putc_t)(int c, FILE * fp); + int (*putc)(int c, FILE * fp) = nullptr; + typedef int (*putw_t)(int w, FILE * fp); + int (*putw)(int w, FILE * fp) = nullptr; + typedef int (*fputs_t)(const char * s, FILE * stream); + int (*fputs)(const char * s, FILE * stream) = nullptr; + typedef size_t (*fread_t)(void * ptr, size_t size, + size_t nmemb, FILE * stream); + size_t (*fread)(void * ptr, size_t size, + size_t nmemb, FILE * stream) = nullptr; + typedef int (*fgetc_t)(FILE * stream); + int (*fgetc)(FILE * stream) = nullptr; + typedef int (*getc_t)(FILE * stream); + int (*getc)(FILE * stream) = nullptr; + typedef int (*getw_t)(FILE * stream); + int (*getw)(FILE * stream) = nullptr; + typedef char * (*fgets_t)(char * s, int size, FILE * stream); + char * (*fgets)(char * s, int size, FILE * stream) = nullptr; + typedef void (*rewind_t)(FILE * stream); + void (*rewind)(FILE * stream) = nullptr; + typedef int (*fseek_t)(FILE * stream, long offset, int whence); + int (*fseek)(FILE * stream, long offset, int whence) = nullptr; + typedef int (*fseeko_t)(FILE * stream, off_t offset, int whence); + int (*fseeko)(FILE * stream, off_t offset, int whence) = nullptr; + typedef int (*fseeko64_t)(FILE * stream, off64_t offset, int whence); + int (*fseeko64)(FILE * stream, off64_t offset, int whence) = nullptr; + typedef int (*fsetpos_t)(FILE * stream, const fpos_t * pos); + int (*fsetpos)(FILE * stream, const fpos_t * pos) = nullptr; + typedef int (*fsetpos64_t)(FILE * stream, const fpos64_t * pos); + int (*fsetpos64)(FILE * stream, const fpos64_t * pos) = nullptr; + typedef long int (*ftell_t)(FILE * fp); + long int (*ftell)(FILE * fp) = nullptr; + API() { + void *is_intercepted = (void*)dlsym(RTLD_DEFAULT, + "stdio_intercepted"); + if (is_intercepted) { + MPI_Init = (MPI_Init_t)dlsym(RTLD_NEXT, "MPI_Init"); + } else { + MPI_Init = (MPI_Init_t)dlsym(RTLD_DEFAULT, "MPI_Init"); + } + if (MPI_Init == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_Init" << std::endl; + } + if (is_intercepted) { + MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_NEXT, + "MPI_Finalize"); + } else { + MPI_Finalize = (MPI_Finalize_t)dlsym(RTLD_DEFAULT, + "MPI_Finalize"); + } + if (MPI_Finalize == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "MPI_Finalize" << std::endl; + } + if (is_intercepted) { + fopen = (fopen_t)dlsym(RTLD_NEXT, "fopen"); + } else { + fopen = (fopen_t)dlsym(RTLD_DEFAULT, "fopen"); + } + if (fopen == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fopen" << std::endl; + } + if (is_intercepted) { + fopen64 = (fopen64_t)dlsym(RTLD_NEXT, "fopen64"); + } else { + fopen64 = (fopen64_t)dlsym(RTLD_DEFAULT, "fopen64"); + } + if (fopen64 == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fopen64" << std::endl; + } + if (is_intercepted) { + fdopen = (fdopen_t)dlsym(RTLD_NEXT, "fdopen"); + } else { + fdopen = (fdopen_t)dlsym(RTLD_DEFAULT, "fdopen"); + } + if (fdopen == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fdopen" << std::endl; + } + if (is_intercepted) { + freopen = (freopen_t)dlsym(RTLD_NEXT, "freopen"); + } else { + freopen = (freopen_t)dlsym(RTLD_DEFAULT, "freopen"); + } + if (freopen == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "freopen" << std::endl; + } + if (is_intercepted) { + freopen64 = (freopen64_t)dlsym(RTLD_NEXT, "freopen64"); + } else { + freopen64 = (freopen64_t)dlsym(RTLD_DEFAULT, "freopen64"); + } + if (freopen64 == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "freopen64" << std::endl; + } + if (is_intercepted) { + fflush = (fflush_t)dlsym(RTLD_NEXT, "fflush"); + } else { + fflush = (fflush_t)dlsym(RTLD_DEFAULT, "fflush"); + } + if (fflush == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fflush" << std::endl; + } + if (is_intercepted) { + fclose = (fclose_t)dlsym(RTLD_NEXT, "fclose"); + } else { + fclose = (fclose_t)dlsym(RTLD_DEFAULT, "fclose"); + } + if (fclose == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fclose" << std::endl; + } + if (is_intercepted) { + fwrite = (fwrite_t)dlsym(RTLD_NEXT, "fwrite"); + } else { + fwrite = (fwrite_t)dlsym(RTLD_DEFAULT, "fwrite"); + } + if (fwrite == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fwrite" << std::endl; + } + if (is_intercepted) { + fputc = (fputc_t)dlsym(RTLD_NEXT, "fputc"); + } else { + fputc = (fputc_t)dlsym(RTLD_DEFAULT, "fputc"); + } + if (fputc == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fputc" << std::endl; + } + if (is_intercepted) { + fgetpos = (fgetpos_t)dlsym(RTLD_NEXT, "fgetpos"); + } else { + fgetpos = (fgetpos_t)dlsym(RTLD_DEFAULT, "fgetpos"); + } + if (fgetpos == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fgetpos" << std::endl; + } + if (is_intercepted) { + fgetpos64 = (fgetpos64_t)dlsym(RTLD_NEXT, "fgetpos64"); + } else { + fgetpos64 = (fgetpos64_t)dlsym(RTLD_DEFAULT, "fgetpos64"); + } + if (fgetpos64 == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fgetpos64" << std::endl; + } + if (is_intercepted) { + putc = (putc_t)dlsym(RTLD_NEXT, "putc"); + } else { + putc = (putc_t)dlsym(RTLD_DEFAULT, "putc"); + } + if (putc == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "putc" << std::endl; + } + if (is_intercepted) { + putw = (putw_t)dlsym(RTLD_NEXT, "putw"); + } else { + putw = (putw_t)dlsym(RTLD_DEFAULT, "putw"); + } + if (putw == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "putw" << std::endl; + } + if (is_intercepted) { + fputs = (fputs_t)dlsym(RTLD_NEXT, "fputs"); + } else { + fputs = (fputs_t)dlsym(RTLD_DEFAULT, "fputs"); + } + if (fputs == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fputs" << std::endl; + } + if (is_intercepted) { + fread = (fread_t)dlsym(RTLD_NEXT, "fread"); + } else { + fread = (fread_t)dlsym(RTLD_DEFAULT, "fread"); + } + if (fread == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fread" << std::endl; + } + if (is_intercepted) { + fgetc = (fgetc_t)dlsym(RTLD_NEXT, "fgetc"); + } else { + fgetc = (fgetc_t)dlsym(RTLD_DEFAULT, "fgetc"); + } + if (fgetc == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fgetc" << std::endl; + } + if (is_intercepted) { + getc = (getc_t)dlsym(RTLD_NEXT, "getc"); + } else { + getc = (getc_t)dlsym(RTLD_DEFAULT, "getc"); + } + if (getc == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "getc" << std::endl; + } + if (is_intercepted) { + getw = (getw_t)dlsym(RTLD_NEXT, "getw"); + } else { + getw = (getw_t)dlsym(RTLD_DEFAULT, "getw"); + } + if (getw == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "getw" << std::endl; + } + if (is_intercepted) { + fgets = (fgets_t)dlsym(RTLD_NEXT, "fgets"); + } else { + fgets = (fgets_t)dlsym(RTLD_DEFAULT, "fgets"); + } + if (fgets == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fgets" << std::endl; + } + if (is_intercepted) { + rewind = (rewind_t)dlsym(RTLD_NEXT, "rewind"); + } else { + rewind = (rewind_t)dlsym(RTLD_DEFAULT, "rewind"); + } + if (rewind == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "rewind" << std::endl; + } + if (is_intercepted) { + fseek = (fseek_t)dlsym(RTLD_NEXT, "fseek"); + } else { + fseek = (fseek_t)dlsym(RTLD_DEFAULT, "fseek"); + } + if (fseek == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fseek" << std::endl; + } + if (is_intercepted) { + fseeko = (fseeko_t)dlsym(RTLD_NEXT, "fseeko"); + } else { + fseeko = (fseeko_t)dlsym(RTLD_DEFAULT, "fseeko"); + } + if (fseeko == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fseeko" << std::endl; + } + if (is_intercepted) { + fseeko64 = (fseeko64_t)dlsym(RTLD_NEXT, "fseeko64"); + } else { + fseeko64 = (fseeko64_t)dlsym(RTLD_DEFAULT, "fseeko64"); + } + if (fseeko64 == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fseeko64" << std::endl; + } + if (is_intercepted) { + fsetpos = (fsetpos_t)dlsym(RTLD_NEXT, "fsetpos"); + } else { + fsetpos = (fsetpos_t)dlsym(RTLD_DEFAULT, "fsetpos"); + } + if (fsetpos == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fsetpos" << std::endl; + } + if (is_intercepted) { + fsetpos64 = (fsetpos64_t)dlsym(RTLD_NEXT, "fsetpos64"); + } else { + fsetpos64 = (fsetpos64_t)dlsym(RTLD_DEFAULT, "fsetpos64"); + } + if (fsetpos64 == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "fsetpos64" << std::endl; + } + if (is_intercepted) { + ftell = (ftell_t)dlsym(RTLD_NEXT, "ftell"); + } else { + ftell = (ftell_t)dlsym(RTLD_DEFAULT, "ftell"); + } + if (ftell == nullptr) { + LOG(FATAL) << "HERMES Adapter failed to map symbol: " + "ftell" << std::endl; + } + } +}; +} // namespace hermes::adapter::stdio + +#endif // HERMES_ADAPTER_STDIO_H diff --git a/adapter/stdio/stdio.cc b/adapter/stdio/stdio.cc index 9f02e9434..022f130ad 100644 --- a/adapter/stdio/stdio.cc +++ b/adapter/stdio/stdio.cc @@ -10,100 +10,40 @@ * have access to the file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include "stdio/stdio.h" +bool stdio_intercepted = true; #include #include #include "interceptor.cc" -#include "adapter_utils.cc" -#include "stdio/mapper/balanced_mapper.cc" -#include "stdio/metadata_manager.cc" +#include +#include "stdio/real_api.h" +#include "stdio/fs_api.h" -using hermes::adapter::stdio::AdapterStat; -using hermes::adapter::stdio::FileStruct; -using hermes::adapter::stdio::MapperFactory; -using hermes::adapter::stdio::MetadataManager; -using hermes::adapter::stdio::global_flushing_mode; using hermes::adapter::WeaklyCanonical; -using hermes::adapter::ReadGap; +using hermes::adapter::stdio::API; +using hermes::adapter::stdio::StdioFS; +using hermes::adapter::Singleton; +using hermes::adapter::fs::MetadataManager; +using hermes::adapter::fs::SeekMode; +namespace hapi = hermes::api; +namespace stdfs = std::experimental::filesystem; using hermes::u8; using hermes::u64; namespace hapi = hermes::api; -namespace fs = std::experimental::filesystem; - - -/** - * Internal Functions - */ - -size_t perform_file_write(const std::string &filename, size_t offset, - size_t size, u8 *data_ptr) { - LOG(INFO) << "Writing to file: " << filename << " offset: " << offset - << " of size:" << size << "." << std::endl; - INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); - FILE *fh = fopen(filename.c_str(), "r+"); - size_t write_size = 0; - if (fh != nullptr) { - if (fseek(fh, offset, SEEK_SET) == 0) { - write_size = fwrite(data_ptr, sizeof(char), size, fh); - if (fclose(fh) != 0) { - hermes::FailedLibraryCall("fclose"); - } - } else { - hermes::FailedLibraryCall("fseek"); - } - } else { - hermes::FailedLibraryCall("fopen"); - } - INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename); - return write_size; -} - -size_t perform_file_read(const char *filename, size_t file_offset, void *ptr, - size_t ptr_offset, size_t size) { - LOG(INFO) << "Read called for filename from destination: " << filename - << " on offset: " << file_offset << " and size: " << size - << std::endl; - INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); - FILE *fh = fopen(filename, "r"); - size_t read_size = 0; - if (fh != nullptr) { - flock(fileno(fh), LOCK_SH); - if (fseek(fh, file_offset, SEEK_SET) == 0) { - read_size = fread((char *)ptr + ptr_offset, sizeof(char), size, fh); - flock(fileno(fh), LOCK_UN); - if (fclose(fh) != 0) { - hermes::FailedLibraryCall("fclose"); - } - } else { - hermes::FailedLibraryCall("fseek"); - } - } else { - hermes::FailedLibraryCall("fopen"); - } - INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename); - - return read_size; -} +namespace stdfs = std::experimental::filesystem; -static bool PersistEagerly(const std::string &path_str) { - bool result = (INTERCEPTOR_LIST->Persists(path_str) && - global_flushing_mode == FlushingMode::kAsynchronous); - - return result; -} /** * MPI */ int HERMES_DECL(MPI_Init)(int *argc, char ***argv) { - MAP_OR_FAIL(MPI_Init); - int status = real_MPI_Init_(argc, argv); + auto real_api = Singleton::GetInstance(); + int status = real_api->MPI_Init(argc, argv); if (status == 0) { - auto mdm = hermes::adapter::Singleton::GetInstance(); + auto mdm = Singleton::GetInstance(); mdm->InitializeHermes(true); LOG(INFO) << "MPI Init intercepted." << std::endl; } @@ -112,1073 +52,417 @@ int HERMES_DECL(MPI_Init)(int *argc, char ***argv) { int HERMES_DECL(MPI_Finalize)(void) { LOG(INFO) << "MPI Finalize intercepted." << std::endl; - auto mdm = hermes::adapter::Singleton::GetInstance(); + auto real_api = Singleton::GetInstance(); + auto mdm = Singleton::GetInstance(); mdm->FinalizeHermes(); - MAP_OR_FAIL(MPI_Finalize); - int status = real_MPI_Finalize_(); + int status = real_api->MPI_Finalize(); return status; } /** * STDIO */ -FILE *simple_open(FILE *ret, const std::string &user_path, const char *mode) { - std::string path_str = WeaklyCanonical(user_path).string(); - - LOG(INFO) << "Open file for filename " << path_str << " in mode " << mode - << std::endl; - auto mdm = hermes::adapter::Singleton::GetInstance(); - - if (ret) { - auto existing = mdm->Find(ret); - if (!existing.second) { - LOG(INFO) << "File not opened before by adapter" << std::endl; - struct stat st; - int fd = fileno(ret); - int status = fstat(fd, &st); - if (status == 0) { - AdapterStat stat(st); - stat.ref_count = 1; - struct timespec ts; - timespec_get(&ts, TIME_UTC); - stat.st_atim = ts; - stat.st_mtim = ts; - stat.st_ctim = ts; - if (strcmp(mode, "a") == 0 || strcmp(mode, "a+") == 0) { - // FIXME(hari): get current size of bucket from Hermes - stat.st_ptr = stat.st_size; - } - // FIXME(hari): check if this initialization is correct. - mdm->InitializeHermes(); - // TODO(hari): how to pass to hermes to make a private bucket - stat.st_bkid = - std::make_shared(path_str, mdm->GetHermes()); - - if (PersistEagerly(path_str)) { - stat.st_vbkt = - std::make_shared(path_str, mdm->GetHermes()); - auto offset_map = std::unordered_map(); - stat.st_persist = - std::make_shared(path_str, offset_map, false); - stat.st_vbkt->Attach(stat.st_persist.get()); - } - - mdm->Create(ret, stat); - } else { - // TODO(hari): @errorhandling invalid fh. - ret = nullptr; - } - } else { - LOG(INFO) << "File opened before by adapter" << std::endl; - existing.first.ref_count++; - struct timespec ts; - timespec_get(&ts, TIME_UTC); - existing.first.st_atim = ts; - existing.first.st_ctim = ts; - mdm->Update(ret, existing.first); - } - } - - return ret; -} - -FILE *open_internal(const std::string &path_str, const char *mode) { - FILE *ret; - MAP_OR_FAIL(fopen); - ret = real_fopen_(path_str.c_str(), mode); - ret = simple_open(ret, path_str.c_str(), mode); - return ret; -} FILE *reopen_internal(const std::string &user_path, const char *mode, FILE *stream) { + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); + auto mdm = Singleton::GetInstance(); FILE *ret; - MAP_OR_FAIL(freopen); - auto mdm = hermes::adapter::Singleton::GetInstance(); - ret = real_freopen_(user_path.c_str(), mode, stream); - if (!ret) { - return ret; - } else { - std::string path_str = WeaklyCanonical(user_path).string(); - LOG(INFO) << "Reopen file for filename " << path_str << " in mode " << mode - << std::endl; - auto existing = mdm->Find(ret); - if (!existing.second) { - LOG(INFO) << "File not opened before by adapter" << std::endl; - return nullptr; - } else { - LOG(INFO) << "File opened before by adapter" << std::endl; - struct timespec ts; - timespec_get(&ts, TIME_UTC); - existing.first.st_atim = ts; - existing.first.st_ctim = ts; - mdm->Update(ret, existing.first); - } - } - return ret; -} - -void PutWithStdioFallback(AdapterStat &stat, const std::string &blob_name, - const std::string &filename, u8 *data, size_t size, - size_t offset) { - hapi::Context ctx; - const char *hermes_write_only = getenv(kHermesWriteOnlyVar); - - if (hermes_write_only && hermes_write_only[0] == '1') { - // Custom DPE for write-only apps like VPIC - ctx.rr_retry = true; - ctx.disable_swap = true; - } + ret = real_api->freopen(user_path.c_str(), mode, stream); + if (!ret) { return ret; } - hapi::Status status = stat.st_bkid->Put(blob_name, data, size, ctx); - if (status.Failed()) { - LOG(WARNING) << "Failed to Put Blob " << blob_name << " to Bucket " - << filename << ". Falling back to stdio." << std::endl; - perform_file_write(filename, offset, size, data); + File f; f.fh_ = ret; fs_api->_InitFile(f); + std::string path_str = WeaklyCanonical(user_path).string(); + LOG(INFO) << "Reopen file for filename " << path_str << " in mode " << mode + << std::endl; + auto existing = mdm->Find(f); + if (!existing.second) { + LOG(INFO) << "File not opened before by adapter" << std::endl; + return nullptr; } else { - stat.st_blobs.emplace(blob_name); - } -} - -size_t write_internal(AdapterStat &stat, const void *ptr, size_t total_size, - FILE *fp) { - std::shared_ptr bkt = stat.st_bkid; - std::string filename = bkt->GetName(); - - LOG(INFO) << "Write called for filename: " << filename << " on offset: " - << stat.st_ptr << " and size: " << total_size << std::endl; - - size_t ret = 0; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto mapper = MapperFactory().Get(kMapperType); - auto mapping = mapper->map( - FileStruct(mdm->Convert(fp), stat.st_ptr, total_size)); - size_t data_offset = 0; - VLOG(1) << "Mapping for write has " << mapping.size() << " mappings.\n"; - - for (const auto &[finfo, hinfo] : mapping) { - auto index = std::stol(hinfo.blob_name_) - 1; - size_t offset = index * kPageSize; - auto blob_exists = bkt->ContainsBlob(hinfo.blob_name_); - u8 *put_data_ptr = (u8 *)ptr + data_offset; - size_t put_data_ptr_size = finfo.size_; - - if (!blob_exists || hinfo.size_ == kPageSize) { - LOG(INFO) << "Create or Overwrite blob " << hinfo.blob_name_ - << " of size:" << hinfo.size_ << "." << std::endl; - if (hinfo.size_ == kPageSize) { - PutWithStdioFallback(stat, hinfo.blob_name_, filename, put_data_ptr, - put_data_ptr_size, finfo.offset_); - } else if (hinfo.offset_ == 0) { - PutWithStdioFallback(stat, hinfo.blob_name_, filename, put_data_ptr, - put_data_ptr_size, offset); - } else { - hapi::Blob final_data(hinfo.offset_ + hinfo.size_); - - ReadGap(filename, offset, final_data.data(), hinfo.offset_, - hinfo.offset_); - memcpy(final_data.data() + hinfo.offset_, put_data_ptr, - put_data_ptr_size); - PutWithStdioFallback(stat, hinfo.blob_name_, filename, - final_data.data(), final_data.size(), offset); - } - } else { - LOG(INFO) << "Blob " << hinfo.blob_name_ - << " of size:" << hinfo.size_ << " exists." << std::endl; - hapi::Blob temp(0); - auto existing_blob_size = bkt->Get(hinfo.blob_name_, temp); - if (hinfo.offset_ == 0) { - LOG(INFO) << "Blob offset is 0" << std::endl; - if (hinfo.size_ >= existing_blob_size) { - LOG(INFO) << "Overwrite blob " << hinfo.blob_name_ - << " of size:" << hinfo.size_ << "." << std::endl; - PutWithStdioFallback(stat, hinfo.blob_name_, filename, - put_data_ptr, put_data_ptr_size, offset); - } else { - LOG(INFO) << "Update blob " << hinfo.blob_name_ - << " of size:" << existing_blob_size << "." << std::endl; - hapi::Blob existing_data(existing_blob_size); - bkt->Get(hinfo.blob_name_, existing_data); - memcpy(existing_data.data(), put_data_ptr, put_data_ptr_size); - - PutWithStdioFallback(stat, hinfo.blob_name_, filename, - existing_data.data(), existing_data.size(), - offset); - } - } else { - LOG(INFO) << "Blob offset: " << hinfo.offset_ << "." << std::endl; - auto new_size = hinfo.offset_ + hinfo.size_; - hapi::Blob existing_data(existing_blob_size); - bkt->Get(hinfo.blob_name_, existing_data); - bkt->DeleteBlob(hinfo.blob_name_); - if (new_size < existing_blob_size) { - new_size = existing_blob_size; - } - hapi::Blob final_data(new_size); - auto existing_data_cp_size = existing_data.size() >= hinfo.offset_ - ? hinfo.offset_ : existing_data.size(); - memcpy(final_data.data(), existing_data.data(), existing_data_cp_size); - - if (existing_blob_size < hinfo.offset_ + 1) { - ReadGap(filename, offset + existing_data_cp_size, - final_data.data() + existing_data_cp_size, - hinfo.offset_ - existing_blob_size, - hinfo.offset_ + hinfo.size_); - } - memcpy(final_data.data() + hinfo.offset_, put_data_ptr, - put_data_ptr_size); - - if (hinfo.offset_ + hinfo.size_ < existing_blob_size) { - LOG(INFO) << "Retain last portion of blob as Blob is bigger than the " - "update." << std::endl; - auto off_t = hinfo.offset_ + hinfo.size_; - memcpy(final_data.data() + off_t, existing_data.data() + off_t, - existing_blob_size - off_t); - } - - PutWithStdioFallback(stat, hinfo.blob_name_, filename, - final_data.data(), final_data.size(), offset); - } - } - data_offset += finfo.size_; - - if (PersistEagerly(filename)) { - hapi::Trait *trait = stat.st_vbkt->GetTrait(hapi::TraitType::PERSIST); - if (trait) { - hapi::PersistTrait *persist_trait = (hapi::PersistTrait *)trait; - persist_trait->offset_map.emplace(hinfo.blob_name_, offset); - } - - stat.st_vbkt->Link(hinfo.blob_name_, filename); - } - } - stat.st_ptr += data_offset; - stat.st_size = stat.st_size >= stat.st_ptr ? stat.st_size : stat.st_ptr; - - struct timespec ts; - timespec_get(&ts, TIME_UTC); - stat.st_mtim = ts; - stat.st_ctim = ts; - mdm->Update(fp, stat); - ret = data_offset; - - return ret; -} - -size_t read_internal(AdapterStat &stat, void *ptr, size_t total_size, - FILE *fp) { - std::shared_ptr bkt = stat.st_bkid; - auto filename = bkt->GetName(); - LOG(INFO) << "Read called for filename: " << filename << " on offset: " - << stat.st_ptr << " and size: " << total_size << std::endl; - if (stat.st_ptr >= stat.st_size) { - return 0; - } - - size_t ret; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto mapper = MapperFactory().Get(kMapperType); - auto mapping = mapper->map( - FileStruct(mdm->Convert(fp), stat.st_ptr, total_size)); - size_t total_read_size = 0; - VLOG(1) << "Mapping for read has " << mapping.size() << " mapping." - << std::endl; - for (const auto& [finfo, hinfo] : mapping) { - auto blob_exists = bkt->ContainsBlob(hinfo.blob_name_); - hapi::Blob read_data(0); - size_t read_size = 0; - if (blob_exists) { - LOG(INFO) << "Blob exists and need to read from Hermes from blob: " - << hinfo.blob_name_ << "." << std::endl; - auto existing_blob_size = bkt->Get(hinfo.blob_name_, read_data); - - read_data.resize(existing_blob_size); - bkt->Get(hinfo.blob_name_, read_data); - bool contains_blob = existing_blob_size > hinfo.offset_; - if (contains_blob) { - read_size = read_data.size() < hinfo.offset_ + hinfo.size_ - ? existing_blob_size - hinfo.offset_ : hinfo.size_; - LOG(INFO) << "Blob have data and need to read from hemes " - "blob: " - << hinfo.blob_name_ << " offset:" << hinfo.offset_ - << " size:" << read_size << "." << std::endl; - memcpy((char *)ptr + total_read_size, - read_data.data() + hinfo.offset_, read_size); - if (read_size < hinfo.size_) { - contains_blob = true; - } else { - contains_blob = false; - } - } else { - LOG(INFO) << "Blob does not have data and need to read from original " - "filename: " - << filename << " offset:" << finfo.offset_ - << " size:" << finfo.size_ << "." << std::endl; - auto file_read_size = - perform_file_read(filename.c_str(), finfo.offset_, ptr, - total_read_size, hinfo.size_); - read_size += file_read_size; - } - if (contains_blob && fs::exists(filename) && - fs::file_size(filename) >= finfo.offset_ + finfo.size_) { - LOG(INFO) << "Blob does not have data and need to read from original " - "filename: " - << filename << " offset:" << finfo.offset_ + read_size - << " size:" << hinfo.size_ - read_size << "." - << std::endl; - auto new_read_size = perform_file_read(filename.c_str(), finfo.offset_, - ptr, total_read_size + read_size, - hinfo.size_ - read_size); - read_size += new_read_size; - } - } else if (fs::exists(filename) && - fs::file_size(filename) >= finfo.offset_ + finfo.size_) { - LOG(INFO) - << "Blob does not exists and need to read from original filename: " - << filename << " offset:" << finfo.offset_ - << " size:" << finfo.size_ << "." << std::endl; - read_size = perform_file_read(filename.c_str(), finfo.offset_, ptr, - total_read_size, finfo.size_); - } - if (read_size > 0) { - total_read_size += read_size; - } + LOG(INFO) << "File opened before by adapter" << std::endl; + struct timespec ts; + timespec_get(&ts, TIME_UTC); + existing.first.st_atim = ts; + existing.first.st_ctim = ts; + mdm->Update(f, existing.first); } - stat.st_ptr += total_read_size; - - struct timespec ts; - timespec_get(&ts, TIME_UTC); - stat.st_atim = ts; - stat.st_ctim = ts; - mdm->Update(fp, stat); - ret = total_read_size; - return ret; } FILE *HERMES_DECL(fopen)(const char *path, const char *mode) { - FILE *ret; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(path)) { LOG(INFO) << "Intercepting fopen(" << path << ", " << mode << ")\n"; - ret = open_internal(path, mode); + AdapterStat stat; + stat.mode_str = mode; + return fs_api->Open(stat, path).fh_; } else { - MAP_OR_FAIL(fopen); - ret = real_fopen_(path, mode); + return real_api->fopen(path, mode); } - - return ret; } FILE *HERMES_DECL(fopen64)(const char *path, const char *mode) { - FILE *ret; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(path)) { LOG(INFO) << "Intercepting fopen64(" << path << ", " << mode << ")\n"; - ret = open_internal(path, mode); + AdapterStat stat; stat.mode_str = mode; + return fs_api->Open(stat, path).fh_; } else { - MAP_OR_FAIL(fopen64); - ret = real_fopen64_(path, mode); + return real_api->fopen64(path, mode); } - - return ret; } FILE *HERMES_DECL(fdopen)(int fd, const char *mode) { - FILE *ret; - MAP_OR_FAIL(fdopen); - ret = real_fdopen_(fd, mode); + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); + FILE *ret = real_api->fdopen(fd, mode); if (ret && hermes::adapter::IsTracked(ret)) { - std::string path_str = hermes::adapter::GetFilenameFromFD(fd); LOG(INFO) << "Intercepting fdopen(" << fd << ", " << mode << ")\n"; - const int kMaxSize = 0xFFF; - char proclnk[kMaxSize]; - char filename[kMaxSize]; - snprintf(proclnk, kMaxSize, "/proc/self/fd/%d", fd); - size_t r = readlink(proclnk, filename, kMaxSize); - filename[r] = '\0'; - ret = simple_open(ret, filename, mode); + std::string path_str = hermes::adapter::GetFilenameFromFD(fd); + File f; f.fh_ = ret; fs_api->_InitFile(f); + AdapterStat stat; stat.mode_str = mode; + fs_api->Open(stat, f, path_str); } return ret; } FILE *HERMES_DECL(freopen)(const char *path, const char *mode, FILE *stream) { - FILE *ret; + auto real_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(path)) { LOG(INFO) << "Intercepting freopen(" << path << ", " << mode << ", " << stream << ")\n"; - ret = reopen_internal(path, mode, stream); - } else { - MAP_OR_FAIL(freopen); - ret = real_freopen_(path, mode, stream); + return reopen_internal(path, mode, stream); } - - return ret; + return real_api->freopen(path, mode, stream); } FILE *HERMES_DECL(freopen64)(const char *path, const char *mode, FILE *stream) { - FILE *ret; + auto real_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(path)) { LOG(INFO) << "Intercepting freopen64(" << path << ", " << mode << ", " << stream << ")\n"; - ret = reopen_internal(path, mode, stream); - } else { - MAP_OR_FAIL(freopen64); - ret = real_freopen64_(path, mode, stream); + return reopen_internal(path, mode, stream); } - - return ret; + return real_api->freopen64(path, mode, stream); } int HERMES_DECL(fflush)(FILE *fp) { - int ret = -1; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (fp && hermes::adapter::IsTracked(fp)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fp); - if (existing.second) { - LOG(INFO) << "Intercepting fflush(" << fp << ")\n"; - auto filename = existing.first.st_bkid->GetName(); - const auto &blob_names = existing.first.st_blobs; - if (!blob_names.empty() && INTERCEPTOR_LIST->Persists(fp)) { - if (PersistEagerly(filename)) { - existing.first.st_vbkt->WaitForBackgroundFlush(); - } else { - LOG(INFO) << "File handler is opened by adapter." << std::endl; - hapi::Context ctx; - LOG(INFO) << "STDIO-fflush Adapter flushes " << blob_names.size() - << " blobs to filename:" << filename << "." << std::endl; - hapi::VBucket file_vbucket(filename, mdm->GetHermes()); - auto offset_map = std::unordered_map(); - - for (const auto &blob_name : blob_names) { - file_vbucket.Link(blob_name, filename, ctx); - auto page_index = std::stol(blob_name) - 1; - offset_map.emplace(blob_name, page_index * kPageSize); - } - bool flush_synchronously = true; - hapi::PersistTrait persist_trait(filename, offset_map, - flush_synchronously); - file_vbucket.Attach(&persist_trait); - existing.first.st_blobs.clear(); - file_vbucket.Destroy(); - } - } - ret = 0; - } - } else { - MAP_OR_FAIL(fflush); - ret = real_fflush_(fp); + File f; f.fh_ = fp; fs_api->_InitFile(f); + return fs_api->Sync(f, stat_exists); } - - return ret; + return real_api->fflush(fp); } int HERMES_DECL(fclose)(FILE *fp) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp)) { LOG(INFO) << "Intercepting fclose(" << fp << ")\n"; - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fp); - if (existing.second) { - LOG(INFO) << "File handler is opened by adapter." << std::endl; - if (existing.first.ref_count == 1) { - auto persist = INTERCEPTOR_LIST->Persists(fp); - auto filename = existing.first.st_bkid->GetName(); - mdm->Delete(fp); - const auto &blob_names = existing.first.st_blobs; - if (!blob_names.empty() && persist) { - if (PersistEagerly(filename)) { - existing.first.st_vbkt->WaitForBackgroundFlush(); - existing.first.st_vbkt->Destroy(); - } else { - LOG(INFO) << "STDIO Adapter flushes " << blob_names.size() - << " blobs to filename:" << filename << "." << std::endl; - INTERCEPTOR_LIST->hermes_flush_exclusion.insert(filename); - hapi::VBucket file_vbucket(filename, mdm->GetHermes()); - auto offset_map = std::unordered_map(); - - for (const auto &blob_name : blob_names) { - auto status = file_vbucket.Link(blob_name, filename); - if (!status.Failed()) { - auto page_index = std::stol(blob_name) - 1; - offset_map.emplace(blob_name, page_index * kPageSize); - } - } - bool flush_synchronously = true; - hapi::PersistTrait persist_trait(filename, offset_map, - flush_synchronously); - file_vbucket.Attach(&persist_trait); - existing.first.st_blobs.clear(); - INTERCEPTOR_LIST->hermes_flush_exclusion.erase(filename); - file_vbucket.Destroy(); - } - } - existing.first.st_bkid->Destroy(); - mdm->FinalizeHermes(); - } else { - LOG(INFO) << "File handler is opened by more than one fopen.\n"; - existing.first.ref_count--; - struct timespec ts; - timespec_get(&ts, TIME_UTC); - existing.first.st_atim = ts; - existing.first.st_ctim = ts; - mdm->Update(fp, existing.first); - existing.first.st_bkid->Release(); - if (existing.first.st_vbkt) { - existing.first.st_vbkt->Release(); - } - } - } + File f; f.fh_ = fp; fs_api->_InitFile(f); + int ret = fs_api->Close(f, stat_exists); + if (stat_exists) return ret; } - - MAP_OR_FAIL(fclose); - ret = real_fclose_(fp); - - return ret; + return real_api->fclose(fp); } size_t HERMES_DECL(fwrite)(const void *ptr, size_t size, size_t nmemb, FILE *fp) { - size_t ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fp); - if (existing.second) { - LOG(INFO) << "Intercepting fwrite(" << ptr << ", " << size << ", " - << nmemb << ", " << fp << ")\n"; - ret = write_internal(existing.first, ptr, size * nmemb, fp); - } else { - MAP_OR_FAIL(fwrite); - ret = real_fwrite_(ptr, size, nmemb, fp); - } - } else { - MAP_OR_FAIL(fwrite); - ret = real_fwrite_(ptr, size, nmemb, fp); + LOG(INFO) << "Intercepting fwrite(" << ptr << ", " << size << ", " + << nmemb << ", " << fp << ")\n"; + File f; f.fh_ = fp; fs_api->_InitFile(f); IoStatus io_status; + size_t ret = fs_api->Write(f, stat_exists, ptr, size*nmemb, io_status); + if (stat_exists) { return ret; } } - - return ret; + return real_api->fwrite(ptr, size, nmemb, fp); } int HERMES_DECL(fputc)(int c, FILE *fp) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fp); - if (existing.second) { LOG(INFO) << "Intercepting fputc(" << c << ", " << fp << ")\n"; - write_internal(existing.first, &c, 1, fp); - ret = c; - } else { - MAP_OR_FAIL(fputc); - ret = real_fputc_(c, fp); - } - } else { - MAP_OR_FAIL(fputc); - ret = real_fputc_(c, fp); + File f; f.fh_ = fp; fs_api->_InitFile(f); IoStatus io_status; + fs_api->Write(f, stat_exists, &c, 1, io_status); + if (stat_exists) { return c; } } - - return ret; + return real_api->fputc(c, fp); } int HERMES_DECL(fgetpos)(FILE *fp, fpos_t *pos) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp) && pos) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fp); - if (existing.second) { - LOG(INFO) << "Intercept fgetpos." << std::endl; - // TODO(chogan): @portability In the GNU C Library, fpos_t is an opaque - // data structure that contains internal data to represent file offset and - // conversion state information. In other systems, it might have a - // different internal representation. This will need to change to support - // other compilers. - pos->__pos = existing.first.st_ptr; - ret = 0; - } else { - MAP_OR_FAIL(fgetpos); - ret = real_fgetpos_(fp, pos); - } - } else { - MAP_OR_FAIL(fgetpos); - ret = real_fgetpos_(fp, pos); + File f; f.fh_ = fp; fs_api->_InitFile(f); + LOG(INFO) << "Intercept fgetpos." << std::endl; + // TODO(chogan): @portability In the GNU C Library, fpos_t is an opaque + // data structure that contains internal data to represent file offset and + // conversion state information. In other systems, it might have a + // different internal representation. This will need to change to support + // other compilers. + pos->__pos = fs_api->Tell(f, stat_exists); + if (stat_exists) { return 0; } } - - return ret; + return real_api->fgetpos(fp, pos); } int HERMES_DECL(fgetpos64)(FILE *fp, fpos64_t *pos) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp) && pos) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fp); - if (existing.second) { - LOG(INFO) << "Intercept fgetpos64." << std::endl; - // TODO(chogan): @portability In the GNU C Library, fpos_t is an opaque - // data structure that contains internal data to represent file offset and - // conversion state information. In other systems, it might have a - // different internal representation. This will need to change to support - // other compilers. - pos->__pos = existing.first.st_ptr; - ret = 0; - } else { - MAP_OR_FAIL(fgetpos64); - ret = real_fgetpos64_(fp, pos); - } - } else { - MAP_OR_FAIL(fgetpos64); - ret = real_fgetpos64_(fp, pos); + File f; f.fh_ = fp; fs_api->_InitFile(f); + LOG(INFO) << "Intercept fgetpos64." << std::endl; + // TODO(chogan): @portability In the GNU C Library, fpos_t is an opaque + // data structure that contains internal data to represent file offset and + // conversion state information. In other systems, it might have a + // different internal representation. This will need to change to support + // other compilers. + pos->__pos = fs_api->Tell(f, stat_exists); + if (stat_exists) { return 0; } } - - return ret; + return real_api->fgetpos64(fp, pos); } int HERMES_DECL(putc)(int c, FILE *fp) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fp); - if (existing.second) { - LOG(INFO) << "Intercept putc." << std::endl; - write_internal(existing.first, &c, 1, fp); - ret = c; - } else { - MAP_OR_FAIL(fputc); - ret = real_fputc_(c, fp); - } - } else { - MAP_OR_FAIL(fputc); - ret = real_fputc_(c, fp); + File f; f.fh_ = fp; fs_api->_InitFile(f); IoStatus io_status; + LOG(INFO) << "Intercept putc." << std::endl; + fs_api->Write(f, stat_exists, &c, 1, io_status); + if (stat_exists) { return c; } } - - return ret; + return real_api->fputc(c, fp); } int HERMES_DECL(putw)(int w, FILE *fp) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fp); - if (existing.second) { - LOG(INFO) << "Intercept putw." << std::endl; - ret = write_internal(existing.first, &w, sizeof(w), fp); - if (ret == sizeof(w)) { - ret = 0; - } else { - ret = EOF; - } + LOG(INFO) << "Intercept putw." << std::endl; + File f; f.fh_ = fp; fs_api->_InitFile(f); IoStatus io_status; + int ret = fs_api->Write(f, stat_exists, &w, sizeof(w), io_status); + if (ret == sizeof(w)) { + return 0; } else { - MAP_OR_FAIL(putw); - ret = real_putw_(w, fp); + return EOF; } - } else { - MAP_OR_FAIL(putw); - ret = real_putw_(w, fp); } - - return ret; + return real_api->putw(w, fp); } int HERMES_DECL(fputs)(const char *s, FILE *stream) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(stream); - if (existing.second) { - LOG(INFO) << "Intercept fputs." << std::endl; - ret = write_internal(existing.first, s, strlen(s), stream); - } else { - MAP_OR_FAIL(fputs); - ret = real_fputs_(s, stream); - } - } else { - MAP_OR_FAIL(fputs); - ret = real_fputs_(s, stream); + LOG(INFO) << "Intercept fputs." << std::endl; + File f; f.fh_ = stream; fs_api->_InitFile(f); IoStatus io_status; + int ret = fs_api->Write(f, stat_exists, s, strlen(s), io_status); + if (stat_exists) { return ret; } } - - return ret; + return real_api->fputs(s, stream);; } size_t HERMES_DECL(fread)(void *ptr, size_t size, size_t nmemb, FILE *stream) { - size_t ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(stream); - if (existing.second) { - LOG(INFO) << "Intercept fread with size: " << size << "." << std::endl; - ret = read_internal(existing.first, ptr, size * nmemb, stream); - } else { - MAP_OR_FAIL(fread); - ret = real_fread_(ptr, size, nmemb, stream); - } - } else { - MAP_OR_FAIL(fread); - ret = real_fread_(ptr, size, nmemb, stream); + LOG(INFO) << "Intercept fread with size: " << size << "." << std::endl; + File f; f.fh_ = stream; fs_api->_InitFile(f); IoStatus io_status; + size_t ret = fs_api->Read(f, stat_exists, ptr, size * nmemb, io_status); + if (stat_exists) { return ret; } } - - return ret; + return real_api->fread(ptr, size, nmemb, stream); } int HERMES_DECL(fgetc)(FILE *stream) { - int ret = -1; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(stream); - if (existing.second) { - LOG(INFO) << "Intercept fgetc." << std::endl; - unsigned char value; - auto ret_size = - read_internal(existing.first, &value, sizeof(unsigned char), stream); - if (ret_size == sizeof(unsigned char)) { - ret = value; - } - } else { - MAP_OR_FAIL(fgetc); - ret = real_fgetc_(stream); - } - } else { - MAP_OR_FAIL(fgetc); - ret = real_fgetc_(stream); + LOG(INFO) << "Intercept fgetc." << std::endl; + File f; f.fh_ = stream; fs_api->_InitFile(f); IoStatus io_status; + u8 value; + fs_api->Read(f, stat_exists, &value, sizeof(u8), io_status); + if (stat_exists) { return value; } } - - return ret; + return real_api->fgetc(stream); } int HERMES_DECL(getc)(FILE *stream) { - int ret = -1; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(stream); - if (existing.second) { - LOG(INFO) << "Intercept getc." << std::endl; - unsigned char value; - auto ret_size = - read_internal(existing.first, &value, sizeof(unsigned char), stream); - if (ret_size == sizeof(unsigned char)) { - ret = value; - } - } else { - MAP_OR_FAIL(fgetc); - ret = real_fgetc_(stream); - } - } else { - MAP_OR_FAIL(fgetc); - ret = real_fgetc_(stream); + LOG(INFO) << "Intercept getc." << std::endl; + File f; f.fh_ = stream; fs_api->_InitFile(f); IoStatus io_status; + u8 value; + fs_api->Read(f, stat_exists, &value, sizeof(u8), io_status); + if (stat_exists) { return value; } } - - return ret; + return real_api->getc(stream); } int HERMES_DECL(getw)(FILE *stream) { - int ret = -1; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(stream); - if (existing.second) { - LOG(INFO) << "Intercept getw." << std::endl; - int value; - auto ret_size = read_internal(existing.first, &value, sizeof(int), - stream); - if (ret_size == sizeof(int)) { - ret = value; - } - } else { - MAP_OR_FAIL(getw); - ret = real_getw_(stream); - } - } else { - MAP_OR_FAIL(getw); - ret = real_getw_(stream); + LOG(INFO) << "Intercept getw." << std::endl; + File f; f.fh_ = stream; fs_api->_InitFile(f); IoStatus io_status; + int value; + fs_api->Read(f, stat_exists, &value, sizeof(int), io_status); + if (stat_exists) { return value; } } - - return ret; + return real_api->getc(stream); } char *HERMES_DECL(fgets)(char *s, int size, FILE *stream) { - char *ret = nullptr; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(stream); - if (existing.second) { - LOG(INFO) << "Intercept fgets." << std::endl; - size_t read_size = size - 1; - size_t ret_size = read_internal(existing.first, s, read_size, stream); - if (ret_size < read_size) { - /* FILE ended */ - read_size = ret_size; - } - /* Check if \0 or \n in s.*/ - size_t copy_pos = 0; - for (size_t i = 0; i < read_size; ++i) { - if (s[i] == '\0' || s[i] == '\n') { - copy_pos = i; - break; - } - } - if (copy_pos > 0) { - /* \n and \0 was found. */ - s[copy_pos + 1] = '\0'; - } else { - s[read_size] = '\0'; + LOG(INFO) << "Intercept fgets." << std::endl; + File f; f.fh_ = stream; fs_api->_InitFile(f); IoStatus io_status; + size_t read_size = size - 1; + size_t ret_size = fs_api->Read(f, stat_exists, s, read_size, io_status); + if (ret_size < read_size) { + /* FILE ended */ + read_size = ret_size; + } + /* Check if \0 or \n in s.*/ + size_t copy_pos = 0; + for (size_t i = 0; i < read_size; ++i) { + if (s[i] == '\0' || s[i] == '\n') { + copy_pos = i; + break; } - ret = s; + } + if (copy_pos > 0) { + /* \n and \0 was found. */ + s[copy_pos + 1] = '\0'; } else { - MAP_OR_FAIL(fgets); - ret = real_fgets_(s, size, stream); + s[read_size] = '\0'; } - } else { - MAP_OR_FAIL(fgets); - ret = real_fgets_(s, size, stream); + if (stat_exists) return s; } - - return ret; + return real_api->fgets(s, size, stream); } void HERMES_DECL(rewind)(FILE *stream) { + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(stream); - if (existing.second) { - LOG(INFO) << "Intercept rewind." << std::endl; - if (!(existing.first.st_mode & O_APPEND)) { - existing.first.st_ptr = 0; - mdm->Update(stream, existing.first); - } else { - LOG(INFO) - << "File pointer not updating as file was opened in append mode." - << std::endl; - } - } else { - MAP_OR_FAIL(rewind); - real_rewind_(stream); - } - } else { - MAP_OR_FAIL(rewind); - real_rewind_(stream); + LOG(INFO) << "Intercept rewind." << std::endl; + File f; f.fh_ = stream; fs_api->_InitFile(f); + fs_api->Seek(f, stat_exists, SeekMode::kSet, 0); + if (stat_exists) { return; } } + real_api->rewind(stream); } int HERMES_DECL(fseek)(FILE *stream, long offset, int whence) { - int ret = -1; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(stream); - if (existing.second) { - LOG(INFO) << "Intercept fseek offset:" << offset << " whence:" << whence - << "." << std::endl; - if (!(existing.first.st_mode & O_APPEND)) { - switch (whence) { - case SEEK_SET: { - existing.first.st_ptr = offset; - break; - } - case SEEK_CUR: { - existing.first.st_ptr += offset; - break; - } - case SEEK_END: { - existing.first.st_ptr = existing.first.st_size + offset; - break; - } - default: { - // TODO(hari): throw not implemented error. - } - } - mdm->Update(stream, existing.first); - ret = 0; - } else { - LOG(INFO) - << "File pointer not updating as file was opened in append mode." - << std::endl; - ret = -1; - } - } else { - MAP_OR_FAIL(fseek); - ret = real_fseek_(stream, offset, whence); - } - } else { - MAP_OR_FAIL(fseek); - ret = real_fseek_(stream, offset, whence); + LOG(INFO) << "Intercept fseek offset:" << offset << " whence:" << whence + << "." << std::endl; + File f; f.fh_ = stream; fs_api->_InitFile(f); + off_t ret = fs_api->Seek(f, stat_exists, + static_cast(whence), offset); + if (stat_exists && ret > 0) { return 0; } } - - return ret; + return real_api->fseek(stream, offset, whence); } int HERMES_DECL(fseeko)(FILE *stream, off_t offset, int whence) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(stream); - if (existing.second) { - LOG(INFO) << "Intercept fseeko offset:" << offset << " whence:" << whence - << "." << std::endl; - if (!(existing.first.st_mode & O_APPEND)) { - switch (whence) { - case SEEK_SET: { - existing.first.st_ptr = offset; - break; - } - case SEEK_CUR: { - existing.first.st_ptr += offset; - break; - } - case SEEK_END: { - existing.first.st_ptr = existing.first.st_size + offset; - break; - } - default: { - // TODO(hari): @errorhandling throw not implemented error. - } - } - mdm->Update(stream, existing.first); - ret = 0; - } else { - LOG(INFO) - << "File pointer not updating as file was opened in append mode." - << std::endl; - ret = -1; - } - } else { - MAP_OR_FAIL(fseeko); - ret = real_fseeko_(stream, offset, whence); - } - } else { - MAP_OR_FAIL(fseeko); - ret = real_fseeko_(stream, offset, whence); + LOG(INFO) << "Intercept fseeko offset:" << offset << " whence:" << whence + << "." << std::endl; + File f; f.fh_ = stream; fs_api->_InitFile(f); + off_t ret = fs_api->Seek(f, stat_exists, + static_cast(whence), offset); + if (stat_exists && ret > 0) { return 0; } } - - return ret; + return real_api->fseeko(stream, offset, whence); } int HERMES_DECL(fseeko64)(FILE *stream, off64_t offset, int whence) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(stream)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(stream); - if (existing.second) { - LOG(INFO) << "Intercept fseeko64 offset:" << offset - << " whence:" << whence << "." << std::endl; - if (!(existing.first.st_mode & O_APPEND)) { - switch (whence) { - case SEEK_SET: { - existing.first.st_ptr = offset; - break; - } - case SEEK_CUR: { - existing.first.st_ptr += offset; - break; - } - case SEEK_END: { - existing.first.st_ptr = existing.first.st_size + offset; - break; - } - default: { - // TODO(hari): throw not implemented error. - } - } - mdm->Update(stream, existing.first); - ret = 0; - } else { - LOG(INFO) - << "File pointer not updating as file was opened in append mode." - << std::endl; - ret = -1; - } - } else { - MAP_OR_FAIL(fseeko64); - ret = real_fseeko64_(stream, offset, whence); - } - } else { - MAP_OR_FAIL(fseeko64); - ret = real_fseeko64_(stream, offset, whence); + LOG(INFO) << "Intercept fseeko offset:" << offset << " whence:" << whence + << "." << std::endl; + File f; f.fh_ = stream; fs_api->_InitFile(f); + off_t ret = fs_api->Seek(f, stat_exists, + static_cast(whence), offset); + if (stat_exists && ret > 0) { return 0; } } - - return ret; + return real_api->fseeko64(stream, offset, whence); } int HERMES_DECL(fsetpos)(FILE *stream, const fpos_t *pos) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); + off_t offset = pos->__pos; if (hermes::adapter::IsTracked(stream)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(stream); - if (existing.second) { - LOG(INFO) << "Intercept fsetpos offset:" << pos->__pos << "." - << std::endl; - if (!(existing.first.st_mode & O_APPEND)) { - // TODO(chogan): @portability In the GNU C Library, fpos_t is an opaque - // data structure that contains internal data to represent file offset - // and conversion state information. In other systems, it might have a - // different internal representation. This will need to change to - // support other compilers. - existing.first.st_ptr = pos->__pos; - mdm->Update(stream, existing.first); - ret = 0; - } else { - LOG(INFO) - << "File pointer not updating as file was opened in append mode." - << std::endl; - ret = -1; - } - } else { - MAP_OR_FAIL(fsetpos); - ret = real_fsetpos_(stream, pos); - } - } else { - MAP_OR_FAIL(fsetpos); - ret = real_fsetpos_(stream, pos); + LOG(INFO) << "Intercept fsetpos offset:" << offset << "." << std::endl; + File f; f.fh_ = stream; fs_api->_InitFile(f); + off_t ret = fs_api->Seek(f, stat_exists, SeekMode::kSet, offset); + if (stat_exists && ret > 0) { return 0; } } - - return ret; + return real_api->fsetpos(stream, pos); } int HERMES_DECL(fsetpos64)(FILE *stream, const fpos64_t *pos) { - int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); + off_t offset = pos->__pos; if (hermes::adapter::IsTracked(stream)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(stream); - if (existing.second) { - LOG(INFO) << "Intercept fsetpos64 offset:" << pos->__pos << "." - << std::endl; - if (!(existing.first.st_mode & O_APPEND)) { - // TODO(chogan): @portability In the GNU C Library, fpos_t is an opaque - // data structure that contains internal data to represent file offset - // and conversion state information. In other systems, it might have a - // different internal representation. This will need to change to - // support other compilers. - existing.first.st_ptr = pos->__pos; - mdm->Update(stream, existing.first); - ret = 0; - } else { - LOG(INFO) - << "File pointer not updating as file was opened in append mode." - << std::endl; - ret = -1; - } - } else { - MAP_OR_FAIL(fsetpos64); - ret = real_fsetpos64_(stream, pos); - } - } else { - MAP_OR_FAIL(fsetpos64); - ret = real_fsetpos64_(stream, pos); + LOG(INFO) << "Intercept fsetpos64 offset:" << offset << "." << std::endl; + File f; f.fh_ = stream; fs_api->_InitFile(f); + off_t ret = fs_api->Seek(f, stat_exists, SeekMode::kSet, offset); + if (stat_exists && ret > 0) { return 0; } } - - return ret; + return real_api->fsetpos64(stream, pos); } long int HERMES_DECL(ftell)(FILE *fp) { - long int ret; + bool stat_exists; + auto real_api = Singleton::GetInstance(); + auto fs_api = Singleton::GetInstance(); if (hermes::adapter::IsTracked(fp)) { - auto mdm = hermes::adapter::Singleton::GetInstance(); - auto existing = mdm->Find(fp); - if (existing.second) { - LOG(INFO) << "Intercept ftell." << std::endl; - ret = existing.first.st_ptr; - } else { - MAP_OR_FAIL(ftell); - ret = real_ftell_(fp); - } - } else { - MAP_OR_FAIL(ftell); - ret = real_ftell_(fp); + LOG(INFO) << "Intercept ftell." << std::endl; + File f; f.fh_ = fp; fs_api->_InitFile(f); + off_t ret = fs_api->Tell(f, stat_exists); + if (stat_exists) { return ret; } } - - return ret; + return real_api->ftell(fp); } diff --git a/adapter/stdio/stdio.h b/adapter/stdio/stdio.h deleted file mode 100644 index aa0d989ad..000000000 --- a/adapter/stdio/stdio.h +++ /dev/null @@ -1,89 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Distributed under BSD 3-Clause license. * - * Copyright by The HDF Group. * - * Copyright by the Illinois Institute of Technology. * - * All rights reserved. * - * * - * This file is part of Hermes. The full Hermes copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the top directory. If you do not * - * have access to the file, you may request a copy from help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef HERMES_ADAPTER_STDIO_H -#define HERMES_ADAPTER_STDIO_H - -/** - * Standard header - */ -#include -#include -#include -#include - -#include - -/** - * Dependent library headers - */ -#include - -/** - * Internal headers - */ -#include "api/hermes.h" -#include "api/bucket.h" -#include "api/vbucket.h" -#include "constants.h" -#include "interceptor.h" -#include "singleton.h" -#include "stdio/datastructures.h" -#include "stdio/mapper/mapper_factory.h" -#include "stdio/metadata_manager.h" - -/** - * Function declarations - */ -HERMES_FORWARD_DECL(ftell, long int, (FILE * fp)); -HERMES_FORWARD_DECL(fopen, FILE *, (const char *path, const char *mode)); -HERMES_FORWARD_DECL(fopen64, FILE *, (const char *path, const char *mode)); -HERMES_FORWARD_DECL(fdopen, FILE *, (int fd, const char *mode)); -HERMES_FORWARD_DECL(freopen, FILE *, - (const char *path, const char *mode, FILE *stream)); -HERMES_FORWARD_DECL(freopen64, FILE *, - (const char *path, const char *mode, FILE *stream)); -HERMES_FORWARD_DECL(fclose, int, (FILE * fp)); -HERMES_FORWARD_DECL(fflush, int, (FILE * fp)); -HERMES_FORWARD_DECL(fwrite, size_t, - (const void *ptr, size_t size, size_t nmemb, FILE *stream)); -HERMES_FORWARD_DECL(fputc, int, (int c, FILE *stream)); -HERMES_FORWARD_DECL(putc, int, (int c, FILE *stream)); -HERMES_FORWARD_DECL(fgetpos, int, (FILE * stream, fpos_t *pos)); -HERMES_FORWARD_DECL(fgetpos64, int, (FILE * stream, fpos64_t *pos)); -HERMES_FORWARD_DECL(putw, int, (int w, FILE *stream)); -HERMES_FORWARD_DECL(fputs, int, (const char *s, FILE *stream)); -HERMES_FORWARD_DECL(fprintf, int, (FILE * stream, const char *format, ...)); -HERMES_FORWARD_DECL(printf, int, (const char *format, ...)); -HERMES_FORWARD_DECL(vfprintf, int, - (FILE * stream, const char *format, va_list)); -HERMES_FORWARD_DECL(vprintf, int, (const char *format, va_list)); -HERMES_FORWARD_DECL(fread, size_t, - (void *ptr, size_t size, size_t nmemb, FILE *stream)); -HERMES_FORWARD_DECL(fgetc, int, (FILE * stream)); -HERMES_FORWARD_DECL(getc, int, (FILE * stream)); -HERMES_FORWARD_DECL(getw, int, (FILE * stream)); -HERMES_FORWARD_DECL(fgets, char *, (char *s, int size, FILE *stream)); -HERMES_FORWARD_DECL(fseek, int, (FILE * stream, long offset, int whence)); -HERMES_FORWARD_DECL(fseeko, int, (FILE * stream, off_t offset, int whence)); -HERMES_FORWARD_DECL(fseeko64, int, (FILE * stream, off64_t offset, int whence)); -HERMES_FORWARD_DECL(fsetpos, int, (FILE * stream, const fpos_t *pos)); -HERMES_FORWARD_DECL(fsetpos64, int, (FILE * stream, const fpos64_t *pos)); -HERMES_FORWARD_DECL(rewind, void, (FILE * stream)); - -/** - * MPI functions declarations - */ -HERMES_FORWARD_DECL(MPI_Init, int, (int *argc, char ***argv)); -HERMES_FORWARD_DECL(MPI_Finalize, int, (void)); - -#endif // HERMES_ADAPTER_STDIO_H diff --git a/adapter/test/data/hermes_small.yaml b/adapter/test/data/hermes_small.yaml index 460b6b5fa..d72ad420a 100644 --- a/adapter/test/data/hermes_small.yaml +++ b/adapter/test/data/hermes_small.yaml @@ -3,7 +3,8 @@ num_devices: 4 num_targets: 4 -capacities_mb: [1, 1, 1, 1] +#2 is used for RAM because some space is taken by the Arena. +capacities_mb: [2, 1, 1, 1] block_sizes_kb: [4, 4, 4, 4] num_slabs: [4, 4, 4, 4] diff --git a/adapter/test/mpiio/CMakeLists.txt b/adapter/test/mpiio/CMakeLists.txt index 86bbc808b..cfd3a2ab1 100644 --- a/adapter/test/mpiio/CMakeLists.txt +++ b/adapter/test/mpiio/CMakeLists.txt @@ -31,3 +31,16 @@ if(HERMES_INSTALL_TESTS) ) endforeach() endif() + +add_executable(mpi_parallel parallel.cc) +add_dependencies(mpi_parallel hermes_mpiio hermes_daemon) +target_link_libraries(mpi_parallel hermes_mpiio Catch2::Catch2 -lstdc++fs -lc MPI::MPI_CXX) +set_target_properties(mpi_parallel PROPERTIES COMPILE_FLAGS "-DHERMES_INTERCEPT=1") + +install( + TARGETS + mpi_parallel + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} +) \ No newline at end of file diff --git a/adapter/test/mpiio/mpiio_adapter_basic_test.cpp b/adapter/test/mpiio/mpiio_adapter_basic_test.cpp index fc9b359e8..a339a9858 100644 --- a/adapter/test/mpiio/mpiio_adapter_basic_test.cpp +++ b/adapter/test/mpiio/mpiio_adapter_basic_test.cpp @@ -92,9 +92,9 @@ TEST_CASE("Open", "[process=" + std::to_string(info.comm_size) + MPI_MODE_WRONLY | MPI_MODE_CREATE | MPI_MODE_DELETE_ON_CLOSE, MPI_COMM_SELF); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::exists(info.new_file.c_str())); + REQUIRE(stdfs::exists(info.new_file.c_str())); test::test_close(); - REQUIRE(!fs::exists(info.new_file.c_str())); + REQUIRE(!stdfs::exists(info.new_file.c_str())); REQUIRE(test::status_orig == MPI_SUCCESS); } posttest(); @@ -180,7 +180,7 @@ TEST_CASE("OpenCollective", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::status_orig == MPI_SUCCESS); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(!fs::exists(info.shared_new_file.c_str())); + REQUIRE(!stdfs::exists(info.shared_new_file.c_str())); } posttest(); } @@ -195,7 +195,7 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + pretest(); bool check_bytes = true; SECTION("write to existing file") { - test::test_open(info.existing_file.c_str(), MPI_MODE_RDWR, MPI_COMM_SELF); + test::test_open(info.existing_file.c_str(), MPI_MODE_RDWR, MPI_COMM_SELF); REQUIRE(test::status_orig == MPI_SUCCESS); test::test_seek(info.rank * args.request_size, MPI_SEEK_SET); REQUIRE(test::status_orig == 0); @@ -213,7 +213,7 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.new_file) == (size_t)test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == (size_t)test::size_written_orig); } SECTION("write to new file with allocate") { @@ -229,12 +229,12 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); MPI_Barrier(MPI_COMM_WORLD); - REQUIRE(fs::file_size(info.shared_new_file) == + REQUIRE(stdfs::file_size(info.shared_new_file) == (size_t)test::size_written_orig * info.comm_size); } SECTION("append to existing file") { - auto existing_size = fs::file_size(info.existing_file); + auto existing_size = stdfs::file_size(info.existing_file); test::test_open(info.existing_file.c_str(), MPI_MODE_WRONLY | MPI_MODE_APPEND | MPI_MODE_EXCL, MPI_COMM_SELF); @@ -243,7 +243,7 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == existing_size + test::size_written_orig); } @@ -256,7 +256,7 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.new_file) == (size_t)test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == (size_t)test::size_written_orig); } SECTION("write_at to existing file") { @@ -278,7 +278,7 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.new_file) == (size_t)test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == (size_t)test::size_written_orig); } SECTION("delete on close mode on new file") { @@ -291,14 +291,14 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::status_orig == 0); test::test_write(info.write_data.c_str(), args.request_size, MPI_CHAR); REQUIRE((size_t)test::size_written_orig == args.request_size); - REQUIRE(fs::exists(info.new_file.c_str())); + REQUIRE(stdfs::exists(info.new_file.c_str())); test::test_close(); - REQUIRE(!fs::exists(info.new_file.c_str())); + REQUIRE(!stdfs::exists(info.new_file.c_str())); REQUIRE(test::status_orig == MPI_SUCCESS); } SECTION("delete on close mode on existing file") { - auto original_size = fs::file_size(info.existing_file); + auto original_size = stdfs::file_size(info.existing_file); test::test_open(info.existing_file.c_str(), MPI_MODE_WRONLY | MPI_MODE_EXCL | MPI_MODE_DELETE_ON_CLOSE, MPI_COMM_SELF); @@ -307,14 +307,14 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::status_orig == 0); test::test_write(info.write_data.c_str(), args.request_size, MPI_CHAR); REQUIRE((size_t)test::size_written_orig == args.request_size); - REQUIRE(fs::exists(info.existing_file.c_str())); + REQUIRE(stdfs::exists(info.existing_file.c_str())); auto new_size = original_size > (size_t)test::size_written_orig * info.comm_size ? original_size : test::size_written_orig * info.comm_size; - REQUIRE(fs::file_size(info.existing_file) == (size_t)new_size); + REQUIRE(stdfs::file_size(info.existing_file) == (size_t)new_size); test::test_close(); - REQUIRE(!fs::exists(info.existing_file.c_str())); + REQUIRE(!stdfs::exists(info.existing_file.c_str())); REQUIRE(test::status_orig == MPI_SUCCESS); check_bytes = false; } @@ -353,26 +353,26 @@ TEST_CASE("SingleWriteCollective", REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.shared_new_file) == + REQUIRE(stdfs::file_size(info.shared_new_file) == (size_t)test::size_written_orig); } // TODO(chogan): This test fails intermittently. Needs diagnosis. // https://github.com/HDFGroup/hermes/issues/209 - // SECTION("write to new file using shared ptr") { - // test::test_open(info.shared_new_file.c_str(), - // MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_COMM_WORLD); - // REQUIRE(test::status_orig == MPI_SUCCESS); - // test::test_seek_shared(0, MPI_SEEK_SET); - // REQUIRE(test::status_orig == 0); - // test::test_write_shared(info.write_data.c_str(), args.request_size, - // MPI_CHAR); - // REQUIRE((size_t)test::size_written_orig == args.request_size); - // test::test_close(); - // REQUIRE(test::status_orig == MPI_SUCCESS); - // REQUIRE(fs::file_size(info.shared_new_file) == - // (size_t)test::size_written_orig * info.comm_size); - // } + SECTION("write to new file using shared ptr") { + test::test_open(info.shared_new_file.c_str(), + MPI_MODE_WRONLY | MPI_MODE_CREATE, MPI_COMM_WORLD); + REQUIRE(test::status_orig == MPI_SUCCESS); + test::test_seek_shared(0, MPI_SEEK_SET); + REQUIRE(test::status_orig == 0); + test::test_write_shared(info.write_data.c_str(), args.request_size, + MPI_CHAR); + REQUIRE((size_t)test::size_written_orig == args.request_size); + test::test_close(); + REQUIRE(test::status_orig == MPI_SUCCESS); + REQUIRE(stdfs::file_size(info.shared_new_file) == + (size_t)test::size_written_orig * info.comm_size); + } SECTION("write to new file with allocate") { test::test_open(info.shared_new_file.c_str(), @@ -387,7 +387,7 @@ TEST_CASE("SingleWriteCollective", test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); MPI_Barrier(MPI_COMM_WORLD); - REQUIRE(fs::file_size(info.shared_new_file) == + REQUIRE(stdfs::file_size(info.shared_new_file) == (size_t)test::size_written_orig * info.comm_size); } @@ -410,11 +410,11 @@ TEST_CASE("SingleWriteCollective", REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.new_file) == (size_t)test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == (size_t)test::size_written_orig); } SECTION("append to existing file") { - auto existing_size = fs::file_size(info.existing_file); + auto existing_size = stdfs::file_size(info.existing_file); test::test_open(info.shared_existing_file.c_str(), MPI_MODE_WRONLY | MPI_MODE_APPEND | MPI_MODE_EXCL, MPI_COMM_WORLD); @@ -423,7 +423,7 @@ TEST_CASE("SingleWriteCollective", REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.shared_existing_file) == + REQUIRE(stdfs::file_size(info.shared_existing_file) == existing_size + test::size_written_orig); } @@ -435,7 +435,7 @@ TEST_CASE("SingleWriteCollective", REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.shared_new_file) == + REQUIRE(stdfs::file_size(info.shared_new_file) == (size_t)test::size_written_orig); } @@ -458,7 +458,7 @@ TEST_CASE("SingleWriteCollective", REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.new_file) == (size_t)test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == (size_t)test::size_written_orig); } SECTION("delete on close mode on new file") { test::test_open( @@ -470,15 +470,15 @@ TEST_CASE("SingleWriteCollective", REQUIRE(test::status_orig == 0); test::test_write(info.write_data.c_str(), args.request_size, MPI_CHAR); REQUIRE((size_t)test::size_written_orig == args.request_size); - REQUIRE(fs::exists(info.shared_new_file.c_str())); + REQUIRE(stdfs::exists(info.shared_new_file.c_str())); MPI_Barrier(MPI_COMM_WORLD); test::test_close(); - REQUIRE(!fs::exists(info.shared_new_file.c_str())); + REQUIRE(!stdfs::exists(info.shared_new_file.c_str())); REQUIRE(test::status_orig == MPI_SUCCESS); } SECTION("delete on close mode on existing file") { - auto original_size = fs::file_size(info.shared_existing_file); + auto original_size = stdfs::file_size(info.shared_existing_file); test::test_open(info.shared_existing_file.c_str(), MPI_MODE_WRONLY | MPI_MODE_EXCL | MPI_MODE_DELETE_ON_CLOSE, MPI_COMM_WORLD); @@ -487,14 +487,14 @@ TEST_CASE("SingleWriteCollective", REQUIRE(test::status_orig == 0); test::test_write(info.write_data.c_str(), args.request_size, MPI_CHAR); REQUIRE((size_t)test::size_written_orig == args.request_size); - REQUIRE(fs::exists(info.shared_existing_file.c_str())); + REQUIRE(stdfs::exists(info.shared_existing_file.c_str())); auto new_size = original_size > (size_t)test::size_written_orig * info.comm_size ? original_size : test::size_written_orig * info.comm_size; - REQUIRE(fs::file_size(info.shared_existing_file) == (size_t)new_size); + REQUIRE(stdfs::file_size(info.shared_existing_file) == (size_t)new_size); test::test_close(); - REQUIRE(!fs::exists(info.shared_existing_file.c_str())); + REQUIRE(!stdfs::exists(info.shared_existing_file.c_str())); REQUIRE(test::status_orig == MPI_SUCCESS); check_bytes = false; } @@ -531,7 +531,7 @@ TEST_CASE("SingleAsyncWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.new_file) == (size_t)test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == (size_t)test::size_written_orig); } SECTION("write to new file using shared ptr") { @@ -545,7 +545,7 @@ TEST_CASE("SingleAsyncWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.shared_new_file) == + REQUIRE(stdfs::file_size(info.shared_new_file) == (size_t)test::size_written_orig * info.comm_size); } @@ -562,12 +562,12 @@ TEST_CASE("SingleAsyncWrite", "[process=" + std::to_string(info.comm_size) + test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); MPI_Barrier(MPI_COMM_WORLD); - REQUIRE(fs::file_size(info.shared_new_file) == + REQUIRE(stdfs::file_size(info.shared_new_file) == (size_t)test::size_written_orig * info.comm_size); } SECTION("append to existing file") { - auto existing_size = fs::file_size(info.existing_file); + auto existing_size = stdfs::file_size(info.existing_file); test::test_open(info.existing_file.c_str(), MPI_MODE_WRONLY | MPI_MODE_APPEND | MPI_MODE_EXCL, MPI_COMM_SELF); @@ -576,7 +576,7 @@ TEST_CASE("SingleAsyncWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == existing_size + test::size_written_orig); } @@ -589,7 +589,7 @@ TEST_CASE("SingleAsyncWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.new_file) == (size_t)test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == (size_t)test::size_written_orig); } SECTION("write_at to existing file") { @@ -611,7 +611,7 @@ TEST_CASE("SingleAsyncWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.new_file) == (size_t)test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == (size_t)test::size_written_orig); } SECTION("delete on close mode on new file") { @@ -624,9 +624,9 @@ TEST_CASE("SingleAsyncWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::status_orig == 0); test::test_iwrite(info.write_data.c_str(), args.request_size, MPI_CHAR); REQUIRE((size_t)test::size_written_orig == args.request_size); - REQUIRE(fs::exists(info.new_file.c_str())); + REQUIRE(stdfs::exists(info.new_file.c_str())); test::test_close(); - REQUIRE(!fs::exists(info.new_file.c_str())); + REQUIRE(!stdfs::exists(info.new_file.c_str())); REQUIRE(test::status_orig == MPI_SUCCESS); } @@ -639,9 +639,9 @@ TEST_CASE("SingleAsyncWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::status_orig == 0); test::test_iwrite(info.write_data.c_str(), args.request_size, MPI_CHAR); REQUIRE((size_t)test::size_written_orig == args.request_size); - REQUIRE(fs::exists(info.existing_file.c_str())); + REQUIRE(stdfs::exists(info.existing_file.c_str())); test::test_close(); - REQUIRE(!fs::exists(info.existing_file.c_str())); + REQUIRE(!stdfs::exists(info.existing_file.c_str())); REQUIRE(test::status_orig == MPI_SUCCESS); check_bytes = false; } @@ -680,7 +680,7 @@ TEST_CASE("SingleAsyncWriteCollective", REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.shared_new_file) == + REQUIRE(stdfs::file_size(info.shared_new_file) == (size_t)test::size_written_orig); } @@ -695,7 +695,7 @@ TEST_CASE("SingleAsyncWriteCollective", REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.shared_new_file) == + REQUIRE(stdfs::file_size(info.shared_new_file) == (size_t)test::size_written_orig * info.comm_size); } @@ -712,7 +712,7 @@ TEST_CASE("SingleAsyncWriteCollective", test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); MPI_Barrier(MPI_COMM_WORLD); - REQUIRE(fs::file_size(info.shared_new_file) == + REQUIRE(stdfs::file_size(info.shared_new_file) == (size_t)test::size_written_orig * info.comm_size); } @@ -735,10 +735,10 @@ TEST_CASE("SingleAsyncWriteCollective", REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.new_file) == (size_t)test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == (size_t)test::size_written_orig); } SECTION("append to existing file") { - auto existing_size = fs::file_size(info.existing_file); + auto existing_size = stdfs::file_size(info.existing_file); test::test_open(info.shared_existing_file.c_str(), MPI_MODE_WRONLY | MPI_MODE_APPEND | MPI_MODE_EXCL, MPI_COMM_WORLD); @@ -747,7 +747,7 @@ TEST_CASE("SingleAsyncWriteCollective", REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.shared_existing_file) == + REQUIRE(stdfs::file_size(info.shared_existing_file) == existing_size + test::size_written_orig); } @@ -759,7 +759,7 @@ TEST_CASE("SingleAsyncWriteCollective", REQUIRE((size_t)test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == MPI_SUCCESS); - REQUIRE(fs::file_size(info.shared_new_file) == + REQUIRE(stdfs::file_size(info.shared_new_file) == (size_t)test::size_written_orig); } SECTION("delete on close mode on new file") { @@ -772,14 +772,14 @@ TEST_CASE("SingleAsyncWriteCollective", REQUIRE(test::status_orig == 0); test::test_iwrite(info.write_data.c_str(), args.request_size, MPI_CHAR); REQUIRE((size_t)test::size_written_orig == args.request_size); - REQUIRE(fs::exists(info.shared_new_file.c_str())); + REQUIRE(stdfs::exists(info.shared_new_file.c_str())); test::test_close(); - REQUIRE(!fs::exists(info.shared_new_file.c_str())); + REQUIRE(!stdfs::exists(info.shared_new_file.c_str())); REQUIRE(test::status_orig == MPI_SUCCESS); } SECTION("delete on close mode on existing file") { - auto original_size = fs::file_size(info.shared_existing_file); + auto original_size = stdfs::file_size(info.shared_existing_file); test::test_open(info.shared_existing_file.c_str(), MPI_MODE_WRONLY | MPI_MODE_EXCL | MPI_MODE_DELETE_ON_CLOSE, MPI_COMM_WORLD); @@ -788,14 +788,14 @@ TEST_CASE("SingleAsyncWriteCollective", REQUIRE(test::status_orig == 0); test::test_iwrite(info.write_data.c_str(), args.request_size, MPI_CHAR); REQUIRE((size_t)test::size_written_orig == args.request_size); - REQUIRE(fs::exists(info.shared_existing_file.c_str())); + REQUIRE(stdfs::exists(info.shared_existing_file.c_str())); auto new_size = original_size > (size_t)test::size_written_orig * info.comm_size ? original_size : test::size_written_orig * info.comm_size; - REQUIRE(fs::file_size(info.shared_existing_file) == (size_t)new_size); + REQUIRE(stdfs::file_size(info.shared_existing_file) == (size_t)new_size); test::test_close(); - REQUIRE(!fs::exists(info.shared_existing_file.c_str())); + REQUIRE(!stdfs::exists(info.shared_existing_file.c_str())); REQUIRE(test::status_orig == MPI_SUCCESS); check_bytes = false; } @@ -992,55 +992,54 @@ TEST_CASE("SingleAsyncRead", "[process=" + std::to_string(info.comm_size) + // TODO(chogan): This test fails sporadically. // https://github.com/HDFGroup/hermes/issues/413 -// TEST_CASE("SingleAsyncReadCollective", -// "[process=" + std::to_string(info.comm_size) + -// "]" -// "[operation=single_read]" -// "[synchronicity=async]" -// "[coordination=collective]" -// "[request_size=type-fixed][repetition=1]" -// "[file=1]") { -// pretest(); -// SECTION("read from non-existing file") { -// test::test_open(info.shared_new_file.c_str(), MPI_MODE_RDONLY, -// MPI_COMM_WORLD); -// REQUIRE(test::status_orig != MPI_SUCCESS); -// } - -// SECTION("read from existing file") { -// test::test_open(info.shared_existing_file.c_str(), MPI_MODE_RDONLY, -// MPI_COMM_WORLD); -// REQUIRE(test::status_orig == MPI_SUCCESS); -// test::test_seek(info.rank * args.request_size, MPI_SEEK_SET); -// REQUIRE(test::status_orig == 0); -// test::test_iread_all(info.read_data.data(), args.request_size, MPI_CHAR); -// REQUIRE((size_t)test::size_read_orig == args.request_size); -// test::test_close(); -// REQUIRE(test::status_orig == MPI_SUCCESS); -// } - -// SECTION("read from existing file using shared ptr") { -// test::test_open(info.shared_existing_file.c_str(), MPI_MODE_RDONLY, -// MPI_COMM_WORLD); -// REQUIRE(test::status_orig == MPI_SUCCESS); -// test::test_seek_shared(0, MPI_SEEK_SET); -// REQUIRE(test::status_orig == 0); -// test::test_iread_shared(info.read_data.data(), args.request_size, -// MPI_CHAR); -// REQUIRE((size_t)test::size_read_orig == args.request_size); -// test::test_close(); -// REQUIRE(test::status_orig == MPI_SUCCESS); -// } - -// SECTION("read_at_all from existing file") { -// test::test_open(info.existing_file.c_str(), MPI_MODE_RDONLY, -// MPI_COMM_SELF); -// REQUIRE(test::status_orig == MPI_SUCCESS); -// test::test_iread_at_all(info.read_data.data(), args.request_size, -// MPI_CHAR, info.rank * args.request_size); -// REQUIRE((size_t)test::size_read_orig == args.request_size); -// test::test_close(); -// REQUIRE(test::status_orig == MPI_SUCCESS); -// } -// posttest(); -// } +TEST_CASE("SingleAsyncReadCollective", + "[process=" + std::to_string(info.comm_size) + + "]" + "[operation=single_read]" + "[synchronicity=async]" + "[coordination=collective]" + "[request_size=type-fixed][repetition=1]" + "[file=1]") { + pretest(); + SECTION("read from non-existing file") { + test::test_open(info.shared_new_file.c_str(), MPI_MODE_RDONLY, + MPI_COMM_WORLD); + REQUIRE(test::status_orig != MPI_SUCCESS); + } + + SECTION("read from existing file") { + test::test_open(info.shared_existing_file.c_str(), MPI_MODE_RDONLY, + MPI_COMM_WORLD); + REQUIRE(test::status_orig == MPI_SUCCESS); + test::test_seek(info.rank * args.request_size, MPI_SEEK_SET); + REQUIRE(test::status_orig == 0); + test::test_iread_all(info.read_data.data(), args.request_size, MPI_CHAR); + REQUIRE((size_t)test::size_read_orig == args.request_size); + test::test_close(); + REQUIRE(test::status_orig == MPI_SUCCESS); + } + + SECTION("read from existing file using shared ptr") { + test::test_open(info.shared_existing_file.c_str(), MPI_MODE_RDONLY, + MPI_COMM_WORLD); + REQUIRE(test::status_orig == MPI_SUCCESS); + test::test_seek_shared(0, MPI_SEEK_SET); + REQUIRE(test::status_orig == 0); + test::test_iread_shared(info.read_data.data(), args.request_size, + MPI_CHAR); + REQUIRE((size_t)test::size_read_orig == args.request_size); + test::test_close(); + REQUIRE(test::status_orig == MPI_SUCCESS); + } + + SECTION("read_at_all from existing file") { + test::test_open(info.existing_file.c_str(), MPI_MODE_RDONLY, MPI_COMM_SELF); + REQUIRE(test::status_orig == MPI_SUCCESS); + test::test_iread_at_all(info.read_data.data(), args.request_size, + MPI_CHAR, info.rank * args.request_size); + REQUIRE((size_t)test::size_read_orig == args.request_size); + test::test_close(); + REQUIRE(test::status_orig == MPI_SUCCESS); + } + posttest(); +} diff --git a/adapter/test/mpiio/mpiio_adapter_test.cpp b/adapter/test/mpiio/mpiio_adapter_test.cpp index 34767c880..0a20a218c 100644 --- a/adapter/test/mpiio/mpiio_adapter_test.cpp +++ b/adapter/test/mpiio/mpiio_adapter_test.cpp @@ -18,12 +18,12 @@ #include "catch_config.h" #include "adapter_test_utils.h" #if HERMES_INTERCEPT == 1 -#include "mpiio/mpiio.h" +#include "filesystem/filesystem.h" #endif #include "adapter_test_utils.h" -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; namespace hermes::adapter::mpiio::test { struct Arguments { @@ -82,7 +82,7 @@ int finalize() { const char* kUser = "USER"; int pretest() { - fs::path fullpath = args.directory; + stdfs::path fullpath = args.directory; fullpath /= args.filename + "_" + std::string(getenv(kUser)); info.new_file = fullpath.string() + "_new_" + std::to_string(getpid()); info.existing_file = fullpath.string() + "_ext_" + std::to_string(getpid()); @@ -98,51 +98,56 @@ int pretest() { fullpath.string() + "_shared_new_cmp_" + std::to_string(info.comm_size); info.shared_existing_file_cmp = fullpath.string() + "_shared_ext_cmp_" + std::to_string(info.comm_size); - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); - if (!fs::exists(info.existing_file)) { + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); + if (!stdfs::exists(info.existing_file)) { std::string cmd = "{ tr -dc '[:alnum:]' < /dev/urandom | head -c " + std::to_string(args.request_size * info.num_iterations) + "; } > " + info.existing_file + " 2> /dev/null"; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == args.request_size * info.num_iterations); - info.total_size = fs::file_size(info.existing_file); + info.total_size = stdfs::file_size(info.existing_file); } MPI_Barrier(MPI_COMM_WORLD); - if (!fs::exists(info.existing_file_cmp)) { + if (!stdfs::exists(info.existing_file_cmp)) { std::string cmd = "cp " + info.existing_file + " " + info.existing_file_cmp; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file_cmp) == + REQUIRE(stdfs::file_size(info.existing_file_cmp) == args.request_size * info.num_iterations); } MPI_Barrier(MPI_COMM_WORLD); if (info.rank == 0) { - if (fs::exists(info.shared_new_file)) fs::remove(info.shared_new_file); - if (fs::exists(info.shared_existing_file)) - fs::remove(info.shared_existing_file); - if (!fs::exists(info.shared_existing_file)) { + if (stdfs::exists(info.shared_new_file)) + stdfs::remove(info.shared_new_file); + if (stdfs::exists(info.shared_existing_file)) + stdfs::remove(info.shared_existing_file); + if (!stdfs::exists(info.shared_existing_file)) { std::string cmd = "cp " + info.existing_file + " " + info.shared_existing_file; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.shared_existing_file) == + REQUIRE(stdfs::file_size(info.shared_existing_file) == args.request_size * info.num_iterations); } - if (fs::exists(info.shared_new_file_cmp)) - fs::remove(info.shared_new_file_cmp); - if (fs::exists(info.shared_existing_file_cmp)) - fs::remove(info.shared_existing_file_cmp); - if (!fs::exists(info.shared_existing_file_cmp)) { + if (stdfs::exists(info.shared_new_file_cmp)) + stdfs::remove(info.shared_new_file_cmp); + if (stdfs::exists(info.shared_existing_file_cmp)) + stdfs::remove(info.shared_existing_file_cmp); + if (!stdfs::exists(info.shared_existing_file_cmp)) { std::string cmd = "cp " + info.existing_file + " " + info.shared_existing_file_cmp; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.shared_existing_file_cmp) == + REQUIRE(stdfs::file_size(info.shared_existing_file_cmp) == args.request_size * info.num_iterations); } } @@ -163,10 +168,10 @@ int posttest(bool compare_data = true) { INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.existing_file); INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.new_file); #endif - if (compare_data && fs::exists(info.new_file) && - fs::exists(info.new_file_cmp)) { - size_t size = fs::file_size(info.new_file); - REQUIRE(size == fs::file_size(info.new_file_cmp)); + if (compare_data && stdfs::exists(info.new_file) && + stdfs::exists(info.new_file_cmp)) { + size_t size = stdfs::file_size(info.new_file); + REQUIRE(size == stdfs::file_size(info.new_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -192,11 +197,11 @@ int posttest(bool compare_data = true) { REQUIRE(char_mismatch == 0); } } - if (compare_data && fs::exists(info.existing_file) && - fs::exists(info.existing_file_cmp)) { - size_t size = fs::file_size(info.existing_file); - if (size != fs::file_size(info.existing_file_cmp)) sleep(1); - REQUIRE(size == fs::file_size(info.existing_file_cmp)); + if (compare_data && stdfs::exists(info.existing_file) && + stdfs::exists(info.existing_file_cmp)) { + size_t size = stdfs::file_size(info.existing_file); + if (size != stdfs::file_size(info.existing_file_cmp)) sleep(1); + REQUIRE(size == stdfs::file_size(info.existing_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -222,10 +227,14 @@ int posttest(bool compare_data = true) { } } /* Clean up. */ - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); #if HERMES_INTERCEPT == 1 INTERCEPTOR_LIST->hermes_flush_exclusion.erase(info.existing_file_cmp); diff --git a/adapter/test/mpiio/parallel.cc b/adapter/test/mpiio/parallel.cc new file mode 100644 index 000000000..9f91a59b5 --- /dev/null +++ b/adapter/test/mpiio/parallel.cc @@ -0,0 +1,46 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "mpi.h" +#include +#include +#include +#include + +namespace stdfs = std::experimental::filesystem; + +int main(int argc, char **argv) { + MPI_File f; + MPI_Status status; + int count = 1024 * 1024 / 8; + int rank, nprocs; + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + std::string path = argv[1]; + std::vector buf(count, rank); + if (rank == 0) { + FILE *fp = fopen(path.c_str(), "w"); + std::vector init(count*nprocs, -1); + fwrite(init.data(), 1, count*nprocs, fp); + fclose(fp); + } + MPI_Barrier(MPI_COMM_WORLD); + + MPI_File_open(MPI_COMM_WORLD, path.c_str(), MPI_MODE_WRONLY | MPI_MODE_CREATE, + MPI_INFO_NULL, &f); + MPI_File_write_at(f, rank*count, buf.data(), count, + MPI_CHAR, &status); + MPI_File_sync(f); + MPI_File_close(&f); + MPI_Finalize(); +} diff --git a/adapter/test/posix/posix_adapter_basic_test.cpp b/adapter/test/posix/posix_adapter_basic_test.cpp index e2ca178f9..ee8ef163c 100644 --- a/adapter/test/posix/posix_adapter_basic_test.cpp +++ b/adapter/test/posix/posix_adapter_basic_test.cpp @@ -66,13 +66,13 @@ TEST_CASE("Open", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::fh_orig != -1); test::test_close(); REQUIRE(test::status_orig == 0); - fs::remove(info.new_file); + stdfs::remove(info.new_file); test::test_open(info.new_file.c_str(), O_RDONLY | O_CREAT, 0600); REQUIRE(test::fh_orig != -1); test::test_close(); REQUIRE(test::status_orig == 0); - fs::remove(info.new_file); + stdfs::remove(info.new_file); test::test_open(info.new_file.c_str(), O_RDWR | O_CREAT, 0600); REQUIRE(test::fh_orig != -1); @@ -258,7 +258,7 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == test::size_written_orig); } SECTION("write to existing file with truncate") { @@ -268,7 +268,7 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.existing_file) == test::size_written_orig); + REQUIRE(stdfs::file_size(info.existing_file) == test::size_written_orig); } SECTION("write to existing file at the end") { @@ -281,19 +281,19 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == test::size_written_orig + args.request_size * info.num_iterations); } SECTION("append to existing file") { - auto existing_size = fs::file_size(info.existing_file); + auto existing_size = stdfs::file_size(info.existing_file); test::test_open(info.existing_file.c_str(), O_RDWR | O_APPEND); REQUIRE(test::fh_orig != -1); test::test_write(info.write_data.data(), args.request_size); REQUIRE(test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == existing_size + test::size_written_orig); } @@ -304,7 +304,7 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::size_written_orig == args.request_size); test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == test::size_written_orig); } posttest(); } @@ -365,7 +365,7 @@ TEST_CASE("BatchedWriteSequential", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == args.request_size); + REQUIRE(stdfs::file_size(info.new_file) == args.request_size); } SECTION("write to new file always at start") { @@ -378,7 +378,7 @@ TEST_CASE("BatchedWriteSequential", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == + REQUIRE(stdfs::file_size(info.new_file) == info.num_iterations * args.request_size); } posttest(); @@ -610,7 +610,7 @@ TEST_CASE("BatchedWriteRSVariable", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == biggest_size_written); + REQUIRE(stdfs::file_size(info.new_file) == biggest_size_written); } SECTION("write to new file") { @@ -627,7 +627,7 @@ TEST_CASE("BatchedWriteRSVariable", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == total_size_written); + REQUIRE(stdfs::file_size(info.new_file) == total_size_written); } posttest(); } @@ -1164,7 +1164,7 @@ TEST_CASE("BatchedWriteTemporalFixed", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == args.request_size); + REQUIRE(stdfs::file_size(info.new_file) == args.request_size); } SECTION("write to new file always at start") { @@ -1178,7 +1178,7 @@ TEST_CASE("BatchedWriteTemporalFixed", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == + REQUIRE(stdfs::file_size(info.new_file) == info.num_iterations * args.request_size); } posttest(); @@ -1249,7 +1249,7 @@ TEST_CASE("BatchedWriteTemporalVariable", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == args.request_size); + REQUIRE(stdfs::file_size(info.new_file) == args.request_size); } SECTION("write to new file always at start") { @@ -1265,7 +1265,7 @@ TEST_CASE("BatchedWriteTemporalVariable", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == + REQUIRE(stdfs::file_size(info.new_file) == info.num_iterations * args.request_size); } posttest(); diff --git a/adapter/test/posix/posix_adapter_mpi_test.cpp b/adapter/test/posix/posix_adapter_mpi_test.cpp index 80dd30e9b..e2201162e 100644 --- a/adapter/test/posix/posix_adapter_mpi_test.cpp +++ b/adapter/test/posix/posix_adapter_mpi_test.cpp @@ -26,10 +26,10 @@ #include "adapter_test_utils.h" #if HERMES_INTERCEPT == 1 -#include "posix/posix.h" +#include "posix/real_api.h" #endif -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; namespace hermes::adapter::posix::test { struct Arguments { @@ -107,7 +107,7 @@ int finalize() { int pretest() { REQUIRE(info.comm_size > 1); - fs::path fullpath = args.directory; + stdfs::path fullpath = args.directory; fullpath /= args.filename; info.new_file = fullpath.string() + "_new_" + std::to_string(info.rank) + "_of_" + std::to_string(info.comm_size) + "_" + @@ -129,64 +129,68 @@ int pretest() { fullpath.string() + "_shared_new_" + std::to_string(info.comm_size); info.shared_new_file_cmp = fullpath.string() + "_shared_new_cmp_" + std::to_string(info.comm_size); - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); - if (fs::exists(info.existing_shared_file)) - fs::remove(info.existing_shared_file); - if (fs::exists(info.existing_shared_file_cmp)) - fs::remove(info.existing_shared_file_cmp); - if (fs::exists(info.shared_new_file)) fs::remove(info.shared_new_file); - if (fs::exists(info.shared_new_file_cmp)) - fs::remove(info.shared_new_file_cmp); - fs::path temp_fullpath = "/tmp"; + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); + if (stdfs::exists(info.existing_shared_file)) + stdfs::remove(info.existing_shared_file); + if (stdfs::exists(info.existing_shared_file_cmp)) + stdfs::remove(info.existing_shared_file_cmp); + if (stdfs::exists(info.shared_new_file)) stdfs::remove(info.shared_new_file); + if (stdfs::exists(info.shared_new_file_cmp)) + stdfs::remove(info.shared_new_file_cmp); + stdfs::path temp_fullpath = "/tmp"; temp_fullpath /= args.filename; std::string temp_ext_file = temp_fullpath.string() + "_temp_" + std::to_string(info.rank) + "_of_" + std::to_string(info.comm_size) + "_" + std::to_string(getpid()); - if (fs::exists(temp_ext_file)) fs::remove(temp_ext_file); - if (!fs::exists(temp_ext_file)) { + if (stdfs::exists(temp_ext_file)) stdfs::remove(temp_ext_file); + if (!stdfs::exists(temp_ext_file)) { std::string cmd = "{ tr -dc '[:alnum:]' < /dev/urandom | head -c " + std::to_string(args.request_size * info.num_iterations) + "; } > " + temp_ext_file + " 2> /dev/null"; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(temp_ext_file) == + REQUIRE(stdfs::file_size(temp_ext_file) == args.request_size * info.num_iterations); - info.total_size = fs::file_size(temp_ext_file); + info.total_size = stdfs::file_size(temp_ext_file); } - if (info.rank == 0 && !fs::exists(info.existing_shared_file)) { + if (info.rank == 0 && !stdfs::exists(info.existing_shared_file)) { std::string cmd = "cp " + temp_ext_file + " " + info.existing_shared_file; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_shared_file) == + REQUIRE(stdfs::file_size(info.existing_shared_file) == args.request_size * info.num_iterations); } - if (info.rank == 0 && !fs::exists(info.existing_shared_file_cmp)) { + if (info.rank == 0 && !stdfs::exists(info.existing_shared_file_cmp)) { std::string cmd = "cp " + temp_ext_file + " " + info.existing_shared_file_cmp; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_shared_file_cmp) == + REQUIRE(stdfs::file_size(info.existing_shared_file_cmp) == args.request_size * info.num_iterations); } - if (!fs::exists(info.existing_file)) { + if (!stdfs::exists(info.existing_file)) { std::string cmd = "cp " + temp_ext_file + " " + info.existing_file; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == args.request_size * info.num_iterations); - info.total_size = fs::file_size(info.existing_file); + info.total_size = stdfs::file_size(info.existing_file); } - if (!fs::exists(info.existing_file_cmp)) { + if (!stdfs::exists(info.existing_file_cmp)) { std::string cmd = "cp " + info.existing_file + " " + info.existing_file_cmp; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file_cmp) == + REQUIRE(stdfs::file_size(info.existing_file_cmp) == args.request_size * info.num_iterations); } - if (fs::exists(temp_ext_file)) fs::remove(temp_ext_file); + if (stdfs::exists(temp_ext_file)) stdfs::remove(temp_ext_file); REQUIRE(info.total_size > 0); MPI_Barrier(MPI_COMM_WORLD); #if HERMES_INTERCEPT == 1 @@ -204,10 +208,10 @@ int posttest(bool compare_data = true) { INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.new_file); INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.existing_shared_file); #endif - if (compare_data && fs::exists(info.new_file) && - fs::exists(info.new_file_cmp)) { - size_t size = fs::file_size(info.new_file); - REQUIRE(size == fs::file_size(info.new_file_cmp)); + if (compare_data && stdfs::exists(info.new_file) && + stdfs::exists(info.new_file_cmp)) { + size_t size = stdfs::file_size(info.new_file); + REQUIRE(size == stdfs::file_size(info.new_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -233,11 +237,11 @@ int posttest(bool compare_data = true) { REQUIRE(char_mismatch == 0); } } - if (compare_data && fs::exists(info.existing_file) && - fs::exists(info.existing_file_cmp)) { - size_t size = fs::file_size(info.existing_file); - if (size != fs::file_size(info.existing_file_cmp)) sleep(1); - REQUIRE(size == fs::file_size(info.existing_file_cmp)); + if (compare_data && stdfs::exists(info.existing_file) && + stdfs::exists(info.existing_file_cmp)) { + size_t size = stdfs::file_size(info.existing_file); + if (size != stdfs::file_size(info.existing_file_cmp)) sleep(1); + REQUIRE(size == stdfs::file_size(info.existing_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -262,11 +266,11 @@ int posttest(bool compare_data = true) { REQUIRE(char_mismatch == 0); } } - if (compare_data && fs::exists(info.existing_shared_file) && - fs::exists(info.existing_shared_file_cmp)) { - size_t size = fs::file_size(info.existing_shared_file); - if (size != fs::file_size(info.existing_shared_file_cmp)) sleep(1); - REQUIRE(size == fs::file_size(info.existing_shared_file_cmp)); + if (compare_data && stdfs::exists(info.existing_shared_file) && + stdfs::exists(info.existing_shared_file_cmp)) { + size_t size = stdfs::file_size(info.existing_shared_file); + if (size != stdfs::file_size(info.existing_shared_file_cmp)) sleep(1); + REQUIRE(size == stdfs::file_size(info.existing_shared_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -292,16 +296,20 @@ int posttest(bool compare_data = true) { } } /* Clean up. */ - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); MPI_Barrier(MPI_COMM_WORLD); if (info.rank == 0) { - if (fs::exists(info.existing_shared_file)) - fs::remove(info.existing_shared_file); - if (fs::exists(info.existing_shared_file_cmp)) - fs::remove(info.existing_shared_file_cmp); + if (stdfs::exists(info.existing_shared_file)) + stdfs::remove(info.existing_shared_file); + if (stdfs::exists(info.existing_shared_file_cmp)) + stdfs::remove(info.existing_shared_file_cmp); } #if HERMES_INTERCEPT == 1 diff --git a/adapter/test/posix/posix_adapter_rs_test.cpp b/adapter/test/posix/posix_adapter_rs_test.cpp index 3f51faf0d..d52d4438d 100644 --- a/adapter/test/posix/posix_adapter_rs_test.cpp +++ b/adapter/test/posix/posix_adapter_rs_test.cpp @@ -36,7 +36,7 @@ TEST_CASE("BatchedWriteRSRangeSmall", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == biggest_size_written); + REQUIRE(stdfs::file_size(info.new_file) == biggest_size_written); } SECTION("write to new file") { @@ -53,7 +53,7 @@ TEST_CASE("BatchedWriteRSRangeSmall", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == total_size_written); + REQUIRE(stdfs::file_size(info.new_file) == total_size_written); } posttest(); } @@ -443,7 +443,7 @@ TEST_CASE("BatchedWriteRSRangeMedium", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == biggest_size_written); + REQUIRE(stdfs::file_size(info.new_file) == biggest_size_written); } SECTION("write to new file") { @@ -460,7 +460,7 @@ TEST_CASE("BatchedWriteRSRangeMedium", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == total_size_written); + REQUIRE(stdfs::file_size(info.new_file) == total_size_written); } posttest(); } @@ -854,7 +854,7 @@ TEST_CASE("BatchedWriteRSRangeLarge", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == biggest_size_written); + REQUIRE(stdfs::file_size(info.new_file) == biggest_size_written); } SECTION("write to new file") { @@ -871,7 +871,7 @@ TEST_CASE("BatchedWriteRSRangeLarge", } test::test_close(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == total_size_written); + REQUIRE(stdfs::file_size(info.new_file) == total_size_written); } posttest(); } diff --git a/adapter/test/posix/posix_adapter_test.cpp b/adapter/test/posix/posix_adapter_test.cpp index 8526396aa..361c0fcf5 100644 --- a/adapter/test/posix/posix_adapter_test.cpp +++ b/adapter/test/posix/posix_adapter_test.cpp @@ -19,7 +19,7 @@ #include "catch_config.h" #if HERMES_INTERCEPT == 1 -#include "posix/posix.h" +#include "posix/real_api.h" #endif #ifndef O_TMPFILE @@ -28,7 +28,7 @@ #include "adapter_test_utils.h" -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; namespace hermes::adapter::posix::test { struct Arguments { @@ -87,7 +87,7 @@ int finalize() { } int pretest() { - fs::path fullpath = args.directory; + stdfs::path fullpath = args.directory; fullpath /= args.filename; info.new_file = fullpath.string() + "_new_" + std::to_string(getpid()); info.existing_file = fullpath.string() + "_ext_" + std::to_string(getpid()); @@ -95,25 +95,29 @@ int pretest() { fullpath.string() + "_new_cmp" + "_" + std::to_string(getpid()); info.existing_file_cmp = fullpath.string() + "_ext_cmp" + "_" + std::to_string(getpid()); - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); - if (!fs::exists(info.existing_file)) { + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); + if (!stdfs::exists(info.existing_file)) { std::string cmd = "{ tr -dc '[:alnum:]' < /dev/urandom | head -c " + std::to_string(args.request_size * info.num_iterations) + "; } > " + info.existing_file + " 2> /dev/null"; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == args.request_size * info.num_iterations); - info.total_size = fs::file_size(info.existing_file); + info.total_size = stdfs::file_size(info.existing_file); } - if (!fs::exists(info.existing_file_cmp)) { + if (!stdfs::exists(info.existing_file_cmp)) { std::string cmd = "cp " + info.existing_file + " " + info.existing_file_cmp; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file_cmp) == + REQUIRE(stdfs::file_size(info.existing_file_cmp) == args.request_size * info.num_iterations); } REQUIRE(info.total_size > 0); @@ -129,10 +133,10 @@ int posttest(bool compare_data = true) { INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.existing_file); INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.new_file); #endif - if (compare_data && fs::exists(info.new_file) && - fs::exists(info.new_file_cmp)) { - size_t size = fs::file_size(info.new_file); - REQUIRE(size == fs::file_size(info.new_file_cmp)); + if (compare_data && stdfs::exists(info.new_file) && + stdfs::exists(info.new_file_cmp)) { + size_t size = stdfs::file_size(info.new_file); + REQUIRE(size == stdfs::file_size(info.new_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -158,11 +162,11 @@ int posttest(bool compare_data = true) { REQUIRE(char_mismatch == 0); } } - if (compare_data && fs::exists(info.existing_file) && - fs::exists(info.existing_file_cmp)) { - size_t size = fs::file_size(info.existing_file); - if (size != fs::file_size(info.existing_file_cmp)) sleep(1); - REQUIRE(size == fs::file_size(info.existing_file_cmp)); + if (compare_data && stdfs::exists(info.existing_file) && + stdfs::exists(info.existing_file_cmp)) { + size_t size = stdfs::file_size(info.existing_file); + if (size != stdfs::file_size(info.existing_file_cmp)) sleep(1); + REQUIRE(size == stdfs::file_size(info.existing_file_cmp)); if (size > 0) { std::vector d1(size, 'r'); std::vector d2(size, 'w'); @@ -191,10 +195,14 @@ int posttest(bool compare_data = true) { } } /* Clean up. */ - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); #if HERMES_INTERCEPT == 1 INTERCEPTOR_LIST->hermes_flush_exclusion.erase(info.existing_file_cmp); diff --git a/adapter/test/stdio/adapter_utils_test.cc b/adapter/test/stdio/adapter_utils_test.cc index dfdb0a82e..c9bd32ef4 100644 --- a/adapter/test/stdio/adapter_utils_test.cc +++ b/adapter/test/stdio/adapter_utils_test.cc @@ -13,7 +13,7 @@ #include "catch_config.h" #include "adapter_utils.h" -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; int init(int* argc, char*** argv) { (void)argc; @@ -36,26 +36,26 @@ TEST_CASE("WeaklyCanonical") { const std::error_code bad_ec = make_error_code(std::errc::invalid_argument); std::error_code ec; - auto dir = fs::path("tmp"); - if (fs::exists(dir)) { - fs::remove_all(dir, ec); + auto dir = stdfs::path("tmp"); + if (stdfs::exists(dir)) { + stdfs::remove_all(dir, ec); } - fs::create_directory(dir); - const auto dirc = fs::canonical(dir); - fs::path foo = dir/"foo", bar = dir/"bar"; - fs::create_directory(foo); - fs::create_directory(bar); - fs::create_directory(bar/"baz"); - fs::path p; + stdfs::create_directory(dir); + const auto dirc = stdfs::canonical(dir); + stdfs::path foo = dir/"foo", bar = dir/"bar"; + stdfs::create_directory(foo); + stdfs::create_directory(bar); + stdfs::create_directory(bar/"baz"); + stdfs::path p; - fs::create_symlink("../bar", foo/"bar"); + stdfs::create_symlink("../bar", foo/"bar"); p = had::WeaklyCanonical(dir/"foo//./bar///../biz/."); REQUIRE(p == dirc/"biz"); p = had::WeaklyCanonical(dir/"foo/.//bar/././baz/."); REQUIRE(p == dirc/"bar/baz"); - p = had::WeaklyCanonical(fs::current_path()/dir/"bar//../foo/bar/baz"); + p = had::WeaklyCanonical(stdfs::current_path()/dir/"bar//../foo/bar/baz"); REQUIRE(p == dirc/"bar/baz"); ec = bad_ec; @@ -67,7 +67,7 @@ TEST_CASE("WeaklyCanonical") { REQUIRE(!ec); REQUIRE(p == dirc/"bar/baz"); ec = bad_ec; - p = had::WeaklyCanonical(fs::current_path()/dir/"bar//../foo/bar/baz", ec); + p = had::WeaklyCanonical(stdfs::current_path()/dir/"bar//../foo/bar/baz", ec); REQUIRE(!ec); REQUIRE(p == dirc/"bar/baz"); @@ -83,7 +83,7 @@ TEST_CASE("WeaklyCanonical") { REQUIRE(p == dirc/"biz"); p = had::WeaklyCanonical(dir/"foo/.././/bar/././baz/."); REQUIRE(p == dirc/"bar/baz"); - p = had::WeaklyCanonical(fs::current_path()/dir/"bar//../foo/../bar/baz"); + p = had::WeaklyCanonical(stdfs::current_path()/dir/"bar//../foo/../bar/baz"); REQUIRE(p == dirc/"bar/baz"); ec = bad_ec; @@ -95,9 +95,10 @@ TEST_CASE("WeaklyCanonical") { REQUIRE(!ec); REQUIRE(p == dirc/"bar/baz"); ec = bad_ec; - p = had::WeaklyCanonical(fs::current_path()/dir/"bar//../foo/../bar/baz", ec); + p = had::WeaklyCanonical( + stdfs::current_path()/dir/"bar//../foo/../bar/baz", ec); REQUIRE(!ec); REQUIRE(p == dirc/"bar/baz"); - fs::remove_all(dir, ec); + stdfs::remove_all(dir, ec); } diff --git a/adapter/test/stdio/stdio_adapter_basic_test.cpp b/adapter/test/stdio/stdio_adapter_basic_test.cpp index ee8ab7dc9..7dacb3f52 100644 --- a/adapter/test/stdio/stdio_adapter_basic_test.cpp +++ b/adapter/test/stdio/stdio_adapter_basic_test.cpp @@ -86,7 +86,7 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::size_written_orig == args.request_size); test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == test::size_written_orig); } SECTION("write to existing file with truncate") { @@ -96,7 +96,7 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::size_written_orig == args.request_size); test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.existing_file) == test::size_written_orig); + REQUIRE(stdfs::file_size(info.existing_file) == test::size_written_orig); } SECTION("write to existing file at the end") { @@ -110,19 +110,19 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::size_written_orig == args.request_size); test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == test::size_written_orig + offset); } SECTION("append to existing file") { - auto existing_size = fs::file_size(info.existing_file); + auto existing_size = stdfs::file_size(info.existing_file); test::test_fopen(info.existing_file.c_str(), "a+"); REQUIRE(test::fh_orig != nullptr); test::test_fwrite(info.write_data.c_str(), args.request_size); REQUIRE(test::size_written_orig == args.request_size); test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == existing_size + test::size_written_orig); } @@ -133,7 +133,7 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(test::size_written_orig == args.request_size); test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == test::size_written_orig); + REQUIRE(stdfs::file_size(info.new_file) == test::size_written_orig); } posttest(); } @@ -197,7 +197,7 @@ TEST_CASE("BatchedWriteSequential", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == args.request_size); + REQUIRE(stdfs::file_size(info.new_file) == args.request_size); } SECTION("write to new file sequentially") { @@ -210,7 +210,7 @@ TEST_CASE("BatchedWriteSequential", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == + REQUIRE(stdfs::file_size(info.new_file) == info.num_iterations * args.request_size); } posttest(); @@ -445,7 +445,7 @@ TEST_CASE("BatchedWriteRSVariable", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == biggest_written); + REQUIRE(stdfs::file_size(info.new_file) == biggest_written); } SECTION("write to new file") { @@ -462,7 +462,7 @@ TEST_CASE("BatchedWriteRSVariable", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == total_test_written); + REQUIRE(stdfs::file_size(info.new_file) == total_test_written); } posttest(); } @@ -1004,7 +1004,7 @@ TEST_CASE("BatchedWriteTemporalFixed", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == args.request_size); + REQUIRE(stdfs::file_size(info.new_file) == args.request_size); } SECTION("write to new file always at start") { @@ -1018,7 +1018,7 @@ TEST_CASE("BatchedWriteTemporalFixed", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == + REQUIRE(stdfs::file_size(info.new_file) == info.num_iterations * args.request_size); } posttest(); @@ -1093,7 +1093,7 @@ TEST_CASE("BatchedWriteTemporalVariable", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == args.request_size); + REQUIRE(stdfs::file_size(info.new_file) == args.request_size); } SECTION("write to new file always at start") { @@ -1109,7 +1109,7 @@ TEST_CASE("BatchedWriteTemporalVariable", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == + REQUIRE(stdfs::file_size(info.new_file) == info.num_iterations * args.request_size); } posttest(); diff --git a/adapter/test/stdio/stdio_adapter_low_buffer_space_test.cpp b/adapter/test/stdio/stdio_adapter_low_buffer_space_test.cpp index d2e1b5495..c66b5fb2a 100644 --- a/adapter/test/stdio/stdio_adapter_low_buffer_space_test.cpp +++ b/adapter/test/stdio/stdio_adapter_low_buffer_space_test.cpp @@ -20,10 +20,10 @@ #include "catch_config.h" #include "adapter_test_utils.h" #if HERMES_INTERCEPT == 1 -#include "stdio/stdio.h" +#include "stdio/real_api.h" #endif -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; namespace hermes::adapter::stdio::test { struct Arguments { @@ -67,7 +67,7 @@ int finalize() { } int pretest() { - fs::path fullpath = args.directory; + stdfs::path fullpath = args.directory; fullpath /= args.filename; info.new_file = fullpath.string() + "_new_" + std::to_string(getpid()); info.existing_file = fullpath.string() + "_ext_" + std::to_string(getpid()); @@ -75,25 +75,29 @@ int pretest() { fullpath.string() + "_new_cmp" + "_" + std::to_string(getpid()); info.existing_file_cmp = fullpath.string() + "_ext_cmp" + "_" + std::to_string(getpid()); - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); - if (!fs::exists(info.existing_file)) { + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); + if (!stdfs::exists(info.existing_file)) { std::string cmd = "{ tr -dc '[:alnum:]' < /dev/urandom | head -c " + std::to_string(args.request_size * info.num_iterations) + "; } > " + info.existing_file + " 2> /dev/null"; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == args.request_size * info.num_iterations); - info.total_size = fs::file_size(info.existing_file); + info.total_size = stdfs::file_size(info.existing_file); } - if (!fs::exists(info.existing_file_cmp)) { + if (!stdfs::exists(info.existing_file_cmp)) { std::string cmd = "cp " + info.existing_file + " " + info.existing_file_cmp; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file_cmp) == + REQUIRE(stdfs::file_size(info.existing_file_cmp) == args.request_size * info.num_iterations); } REQUIRE(info.total_size > 0); @@ -109,10 +113,10 @@ int posttest(bool compare_data = true) { INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.existing_file); INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.new_file); #endif - if (compare_data && fs::exists(info.new_file) && - fs::exists(info.new_file_cmp)) { - size_t size = fs::file_size(info.new_file); - REQUIRE(size == fs::file_size(info.new_file_cmp)); + if (compare_data && stdfs::exists(info.new_file) && + stdfs::exists(info.new_file_cmp)) { + size_t size = stdfs::file_size(info.new_file); + REQUIRE(size == stdfs::file_size(info.new_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -141,11 +145,11 @@ int posttest(bool compare_data = true) { REQUIRE(char_mismatch == 0); } } - if (compare_data && fs::exists(info.existing_file) && - fs::exists(info.existing_file_cmp)) { - size_t size = fs::file_size(info.existing_file); - if (size != fs::file_size(info.existing_file_cmp)) sleep(1); - REQUIRE(size == fs::file_size(info.existing_file_cmp)); + if (compare_data && stdfs::exists(info.existing_file) && + stdfs::exists(info.existing_file_cmp)) { + size_t size = stdfs::file_size(info.existing_file); + if (size != stdfs::file_size(info.existing_file_cmp)) sleep(1); + REQUIRE(size == stdfs::file_size(info.existing_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -171,10 +175,14 @@ int posttest(bool compare_data = true) { } } /* Clean up. */ - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); #if HERMES_INTERCEPT == 1 INTERCEPTOR_LIST->hermes_flush_exclusion.erase(info.existing_file_cmp); @@ -264,7 +272,7 @@ TEST_CASE("BatchedWriteSequential", REQUIRE(test::size_written_orig == write_size); test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == write_size); + REQUIRE(stdfs::file_size(info.new_file) == write_size); } SECTION("write to new file multiple write") { test::test_fopen(info.new_file.c_str(), "w+"); @@ -276,7 +284,7 @@ TEST_CASE("BatchedWriteSequential", REQUIRE(test::size_written_orig == args.request_size); test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == + REQUIRE(stdfs::file_size(info.new_file) == args.request_size * (info.num_iterations + 1)); } posttest(); diff --git a/adapter/test/stdio/stdio_adapter_mapper_test.cpp b/adapter/test/stdio/stdio_adapter_mapper_test.cpp index ed9603204..1a7c31cea 100644 --- a/adapter/test/stdio/stdio_adapter_mapper_test.cpp +++ b/adapter/test/stdio/stdio_adapter_mapper_test.cpp @@ -14,17 +14,15 @@ #include "catch_config.h" #include "constants.h" -#include "stdio/constants.h" -#include "stdio/datastructures.h" -#include "stdio/mapper/mapper_factory.h" -#include "stdio/metadata_manager.h" +#include "stdio/fs_api.h" +#include "mapper/mapper_factory.h" -using hermes::adapter::stdio::FileStruct; -using hermes::adapter::stdio::HermesStruct; -using hermes::adapter::stdio::MapperFactory; -using hermes::adapter::stdio::MetadataManager; +using hermes::adapter::BlobPlacements; +using hermes::adapter::MapperFactory; +using hermes::adapter::fs::MetadataManager; +using hermes::adapter::fs::kMapperType; -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; namespace hermes::adapter::stdio::test { struct Arguments { @@ -62,30 +60,30 @@ int finalize() { } int pretest() { - fs::path fullpath = args.directory; + stdfs::path fullpath = args.directory; fullpath /= args.filename; info.new_file = fullpath.string() + "_new"; info.existing_file = fullpath.string() + "_ext"; - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (!fs::exists(info.existing_file)) { + if (stdfs::exists(info.new_file)) stdfs::remove(info.new_file); + if (stdfs::exists(info.existing_file)) stdfs::remove(info.existing_file); + if (!stdfs::exists(info.existing_file)) { std::string cmd = "dd if=/dev/zero of=" + info.existing_file + " bs=1 count=0 seek=" + std::to_string(args.request_size * args.num_iterations) + " > /dev/null 2>&1"; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == args.request_size * args.num_iterations); - info.total_size = fs::file_size(info.existing_file); + info.total_size = stdfs::file_size(info.existing_file); } REQUIRE(info.total_size > 0); return 0; } int posttest() { - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); + if (stdfs::exists(info.new_file)) stdfs::remove(info.new_file); + if (stdfs::exists(info.existing_file)) stdfs::remove(info.existing_file); return 0; } @@ -107,55 +105,49 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + "[pattern=sequential][file=1]") { pretest(); SECTION("Map a one request") { - auto mdm = hermes::adapter::Singleton::GetInstance(); auto mapper = MapperFactory().Get(kMapperType); size_t total_size = args.request_size; FILE* fp = fopen(info.new_file.c_str(), "w+"); REQUIRE(fp != nullptr); size_t offset = 0; REQUIRE(kPageSize > total_size + offset); - auto mapping = - mapper->map(FileStruct(mdm->Convert(fp), offset, total_size)); + BlobPlacements mapping; + mapper->map(offset, total_size, mapping); REQUIRE(mapping.size() == 1); - REQUIRE(mapping[0].first.offset_ == offset); - REQUIRE(mapping[0].first.size_ == total_size); - REQUIRE(mapping[0].second.offset_ == offset); - REQUIRE(mapping[0].second.size_ == total_size); - + REQUIRE(mapping[0].bucket_off_ == offset); + REQUIRE(mapping[0].blob_size_ == total_size); + REQUIRE(mapping[0].blob_off_ == offset); int status = fclose(fp); REQUIRE(status == 0); } SECTION("Map a one big request") { - auto mdm = hermes::adapter::Singleton::GetInstance(); auto mapper = MapperFactory().Get(kMapperType); size_t total_size = args.request_size * args.num_iterations; FILE* fp = fopen(info.new_file.c_str(), "w+"); REQUIRE(fp != nullptr); size_t offset = 0; - auto mapping = - mapper->map(FileStruct(mdm->Convert(fp), offset, total_size)); + BlobPlacements mapping; + mapper->map(offset, total_size, mapping); REQUIRE(mapping.size() == ceil((double)total_size / kPageSize)); for (const auto& item : mapping) { size_t mapped_size = total_size - offset > kPageSize ? kPageSize : total_size - offset; - REQUIRE(item.first.offset_ == offset); - REQUIRE(item.first.size_ == mapped_size); - REQUIRE(item.second.offset_ == offset % kPageSize); - REQUIRE(item.second.size_ == mapped_size); + REQUIRE(item.bucket_off_ == offset); + REQUIRE(item.blob_size_ == mapped_size); + REQUIRE(item.blob_off_ == offset % kPageSize); offset += mapped_size; } int status = fclose(fp); REQUIRE(status == 0); } SECTION("Map a one large unaligned request") { - auto mdm = hermes::adapter::Singleton::GetInstance(); auto mapper = MapperFactory().Get(kMapperType); size_t total_size = args.request_size * args.num_iterations; FILE* fp = fopen(info.new_file.c_str(), "w+"); REQUIRE(fp != nullptr); size_t offset = 1; - auto mapping = - mapper->map(FileStruct(mdm->Convert(fp), offset, total_size)); + BlobPlacements mapping; + mapper->map(offset, total_size, mapping); bool has_rem = (total_size + offset) % kPageSize != 0; if (has_rem) { REQUIRE(mapping.size() == ceil((double)total_size / kPageSize) + 1); @@ -174,10 +166,9 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + } else { mapped_size = kPageSize; } - REQUIRE(item.first.offset_ == current_offset); - REQUIRE(item.first.size_ == mapped_size); - REQUIRE(item.second.offset_ == current_offset % kPageSize); - REQUIRE(item.second.size_ == mapped_size); + REQUIRE(item.bucket_off_ == current_offset); + REQUIRE(item.blob_size_ == mapped_size); + REQUIRE(item.blob_off_ == current_offset % kPageSize); current_offset += mapped_size; i++; } @@ -185,20 +176,18 @@ TEST_CASE("SingleWrite", "[process=" + std::to_string(info.comm_size) + REQUIRE(status == 0); } SECTION("Map a one small unaligned request") { - auto mdm = hermes::adapter::Singleton::GetInstance(); auto mapper = MapperFactory().Get(kMapperType); size_t total_size = args.request_size; FILE* fp = fopen(info.new_file.c_str(), "w+"); REQUIRE(fp != nullptr); size_t offset = 1; REQUIRE(kPageSize > total_size + offset); - auto mapping = - mapper->map(FileStruct(mdm->Convert(fp), offset, total_size)); + BlobPlacements mapping; + mapper->map(offset, total_size, mapping); REQUIRE(mapping.size() == 1); - REQUIRE(mapping[0].first.offset_ == offset); - REQUIRE(mapping[0].first.size_ == total_size); - REQUIRE(mapping[0].second.offset_ == 1); - REQUIRE(mapping[0].second.size_ == total_size); + REQUIRE(mapping[0].bucket_off_ == offset); + REQUIRE(mapping[0].blob_size_ == total_size); + REQUIRE(mapping[0].blob_off_ == 1); int status = fclose(fp); REQUIRE(status == 0); } diff --git a/adapter/test/stdio/stdio_adapter_mode_test.cpp b/adapter/test/stdio/stdio_adapter_mode_test.cpp index aa3dd3d34..0859ca90a 100644 --- a/adapter/test/stdio/stdio_adapter_mode_test.cpp +++ b/adapter/test/stdio/stdio_adapter_mode_test.cpp @@ -19,12 +19,12 @@ #include #if HERMES_INTERCEPT == 1 -#include "stdio/stdio.h" +#include "stdio/real_api.h" #endif #include "adapter_test_utils.h" -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; namespace hermes::adapter::stdio::test { struct Arguments { @@ -57,7 +57,7 @@ hermes::adapter::stdio::test::Arguments args; hermes::adapter::stdio::test::Info info; int init(int* argc, char*** argv) { - fs::path fullpath = args.directory; + stdfs::path fullpath = args.directory; fullpath /= args.filename; info.new_file = fullpath.string() + "_new" + std::to_string(getpid()); info.existing_file = fullpath.string() + "_ext" + std::to_string(getpid()); @@ -80,25 +80,29 @@ int finalize() { } int pretest() { - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); - if (!fs::exists(info.existing_file)) { + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); + if (!stdfs::exists(info.existing_file)) { std::string cmd = "{ tr -dc '[:alnum:]' < /dev/urandom | head -c " + std::to_string(args.request_size * info.num_iterations) + "; } > " + info.existing_file + " 2> /dev/null"; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == args.request_size * info.num_iterations); - info.total_size = fs::file_size(info.existing_file); + info.total_size = stdfs::file_size(info.existing_file); } - if (!fs::exists(info.existing_file_cmp)) { + if (!stdfs::exists(info.existing_file_cmp)) { std::string cmd = "cp " + info.existing_file + " " + info.existing_file_cmp; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file_cmp) == + REQUIRE(stdfs::file_size(info.existing_file_cmp) == args.request_size * info.num_iterations); } REQUIRE(info.total_size > 0); @@ -114,10 +118,10 @@ int posttest(bool compare_data = true) { INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.existing_file); INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.new_file); #endif - if (compare_data && fs::exists(info.new_file) && - fs::exists(info.new_file_cmp)) { - size_t size = fs::file_size(info.new_file); - REQUIRE(size == fs::file_size(info.new_file_cmp)); + if (compare_data && stdfs::exists(info.new_file) && + stdfs::exists(info.new_file_cmp)) { + size_t size = stdfs::file_size(info.new_file); + REQUIRE(size == stdfs::file_size(info.new_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -143,11 +147,11 @@ int posttest(bool compare_data = true) { REQUIRE(char_mismatch == 0); } } - if (compare_data && fs::exists(info.existing_file) && - fs::exists(info.existing_file_cmp)) { - size_t size = fs::file_size(info.existing_file); - if (size != fs::file_size(info.existing_file_cmp)) sleep(1); - REQUIRE(size == fs::file_size(info.existing_file_cmp)); + if (compare_data && stdfs::exists(info.existing_file) && + stdfs::exists(info.existing_file_cmp)) { + size_t size = stdfs::file_size(info.existing_file); + if (size != stdfs::file_size(info.existing_file_cmp)) sleep(1); + REQUIRE(size == stdfs::file_size(info.existing_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -173,10 +177,14 @@ int posttest(bool compare_data = true) { } } /* Clean up. */ - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); #if HERMES_INTERCEPT == 1 INTERCEPTOR_LIST->hermes_flush_exclusion.erase(info.existing_file_cmp); @@ -273,7 +281,7 @@ TEST_CASE("BatchedWriteSequentialPersistent", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == + REQUIRE(stdfs::file_size(info.new_file) == info.num_iterations * args.request_size); } posttest(); @@ -303,7 +311,7 @@ TEST_CASE("BatchedWriteSequentialBypass", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == + REQUIRE(stdfs::file_size(info.new_file) == info.num_iterations * args.request_size); } posttest(); @@ -333,7 +341,7 @@ TEST_CASE("BatchedWriteSequentialScratch", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == 0); + REQUIRE(stdfs::file_size(info.new_file) == 0); } posttest(false); } diff --git a/adapter/test/stdio/stdio_adapter_mpi_test.cpp b/adapter/test/stdio/stdio_adapter_mpi_test.cpp index 232f996d5..bb2870cf0 100644 --- a/adapter/test/stdio/stdio_adapter_mpi_test.cpp +++ b/adapter/test/stdio/stdio_adapter_mpi_test.cpp @@ -19,10 +19,10 @@ #include #include #if HERMES_INTERCEPT == 1 -#include "stdio/stdio.h" +#include "stdio/real_api.h" #endif -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; namespace hermes::adapter::stdio::test { struct Arguments { @@ -130,7 +130,7 @@ void test_fseek(long offset, int whence) { int pretest() { REQUIRE(info.comm_size > 1); - fs::path fullpath = args.directory; + stdfs::path fullpath = args.directory; fullpath /= args.filename; info.new_file = fullpath.string() + "_new_" + std::to_string(info.rank) + "_of_" + std::to_string(info.comm_size) + "_" + @@ -148,61 +148,65 @@ int pretest() { fullpath.string() + "_ext_" + std::to_string(info.comm_size); info.existing_shared_file_cmp = fullpath.string() + "_ext_cmp_" + std::to_string(info.comm_size); - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); - if (fs::exists(info.existing_shared_file)) - fs::remove(info.existing_shared_file); - if (fs::exists(info.existing_shared_file_cmp)) - fs::remove(info.existing_shared_file_cmp); - fs::path temp_fullpath = "/tmp"; + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); + if (stdfs::exists(info.existing_shared_file)) + stdfs::remove(info.existing_shared_file); + if (stdfs::exists(info.existing_shared_file_cmp)) + stdfs::remove(info.existing_shared_file_cmp); + stdfs::path temp_fullpath = "/tmp"; temp_fullpath /= args.filename; std::string temp_ext_file = temp_fullpath.string() + "_temp_" + std::to_string(info.rank) + "_of_" + std::to_string(info.comm_size) + "_" + std::to_string(getpid()); - if (fs::exists(temp_ext_file)) fs::remove(temp_ext_file); - if (!fs::exists(temp_ext_file)) { + if (stdfs::exists(temp_ext_file)) stdfs::remove(temp_ext_file); + if (!stdfs::exists(temp_ext_file)) { std::string cmd = "{ tr -dc '[:alnum:]' < /dev/urandom | head -c " + std::to_string(args.request_size * info.num_iterations) + "; } > " + temp_ext_file + " 2> /dev/null"; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(temp_ext_file) == + REQUIRE(stdfs::file_size(temp_ext_file) == args.request_size * info.num_iterations); - info.total_size = fs::file_size(temp_ext_file); + info.total_size = stdfs::file_size(temp_ext_file); } - if (info.rank == 0 && !fs::exists(info.existing_shared_file)) { + if (info.rank == 0 && !stdfs::exists(info.existing_shared_file)) { std::string cmd = "cp " + temp_ext_file + " " + info.existing_shared_file; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_shared_file) == + REQUIRE(stdfs::file_size(info.existing_shared_file) == args.request_size * info.num_iterations); } - if (info.rank == 0 && !fs::exists(info.existing_shared_file_cmp)) { + if (info.rank == 0 && !stdfs::exists(info.existing_shared_file_cmp)) { std::string cmd = "cp " + temp_ext_file + " " + info.existing_shared_file_cmp; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_shared_file_cmp) == + REQUIRE(stdfs::file_size(info.existing_shared_file_cmp) == args.request_size * info.num_iterations); } - if (!fs::exists(info.existing_file)) { + if (!stdfs::exists(info.existing_file)) { std::string cmd = "cp " + temp_ext_file + " " + info.existing_file; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == args.request_size * info.num_iterations); - info.total_size = fs::file_size(info.existing_file); + info.total_size = stdfs::file_size(info.existing_file); } - if (!fs::exists(info.existing_file_cmp)) { + if (!stdfs::exists(info.existing_file_cmp)) { std::string cmd = "cp " + info.existing_file + " " + info.existing_file_cmp; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file_cmp) == + REQUIRE(stdfs::file_size(info.existing_file_cmp) == args.request_size * info.num_iterations); } - if (fs::exists(temp_ext_file)) fs::remove(temp_ext_file); + if (stdfs::exists(temp_ext_file)) stdfs::remove(temp_ext_file); REQUIRE(info.total_size > 0); MPI_Barrier(MPI_COMM_WORLD); #if HERMES_INTERCEPT == 1 @@ -220,10 +224,10 @@ int posttest(bool compare_data = true) { INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.new_file); INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.existing_shared_file); #endif - if (compare_data && fs::exists(info.new_file) && - fs::exists(info.new_file_cmp)) { - size_t size = fs::file_size(info.new_file); - REQUIRE(size == fs::file_size(info.new_file_cmp)); + if (compare_data && stdfs::exists(info.new_file) && + stdfs::exists(info.new_file_cmp)) { + size_t size = stdfs::file_size(info.new_file); + REQUIRE(size == stdfs::file_size(info.new_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -249,11 +253,11 @@ int posttest(bool compare_data = true) { REQUIRE(char_mismatch == 0); } } - if (compare_data && fs::exists(info.existing_file) && - fs::exists(info.existing_file_cmp)) { - size_t size = fs::file_size(info.existing_file); - if (size != fs::file_size(info.existing_file_cmp)) sleep(1); - REQUIRE(size == fs::file_size(info.existing_file_cmp)); + if (compare_data && stdfs::exists(info.existing_file) && + stdfs::exists(info.existing_file_cmp)) { + size_t size = stdfs::file_size(info.existing_file); + if (size != stdfs::file_size(info.existing_file_cmp)) sleep(1); + REQUIRE(size == stdfs::file_size(info.existing_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -278,11 +282,11 @@ int posttest(bool compare_data = true) { REQUIRE(char_mismatch == 0); } } - if (compare_data && fs::exists(info.existing_shared_file) && - fs::exists(info.existing_shared_file_cmp)) { - size_t size = fs::file_size(info.existing_shared_file); - if (size != fs::file_size(info.existing_shared_file_cmp)) sleep(1); - REQUIRE(size == fs::file_size(info.existing_shared_file_cmp)); + if (compare_data && stdfs::exists(info.existing_shared_file) && + stdfs::exists(info.existing_shared_file_cmp)) { + size_t size = stdfs::file_size(info.existing_shared_file); + if (size != stdfs::file_size(info.existing_shared_file_cmp)) sleep(1); + REQUIRE(size == stdfs::file_size(info.existing_shared_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -308,16 +312,20 @@ int posttest(bool compare_data = true) { } } /* Clean up. */ - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); MPI_Barrier(MPI_COMM_WORLD); if (info.rank == 0) { - if (fs::exists(info.existing_shared_file)) - fs::remove(info.existing_shared_file); - if (fs::exists(info.existing_shared_file_cmp)) - fs::remove(info.existing_shared_file_cmp); + if (stdfs::exists(info.existing_shared_file)) + stdfs::remove(info.existing_shared_file); + if (stdfs::exists(info.existing_shared_file_cmp)) + stdfs::remove(info.existing_shared_file_cmp); } #if HERMES_INTERCEPT == 1 diff --git a/adapter/test/stdio/stdio_adapter_rs_test.cpp b/adapter/test/stdio/stdio_adapter_rs_test.cpp index 3b22fe678..b6a4ea629 100644 --- a/adapter/test/stdio/stdio_adapter_rs_test.cpp +++ b/adapter/test/stdio/stdio_adapter_rs_test.cpp @@ -38,7 +38,7 @@ TEST_CASE("BatchedWriteRSRangeSmall", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == biggest_written); + REQUIRE(stdfs::file_size(info.new_file) == biggest_written); } SECTION("write to new file") { @@ -55,7 +55,7 @@ TEST_CASE("BatchedWriteRSRangeSmall", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == total_written); + REQUIRE(stdfs::file_size(info.new_file) == total_written); } posttest(); } @@ -450,7 +450,7 @@ TEST_CASE("BatchedWriteRSRangeMedium", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == biggest_written); + REQUIRE(stdfs::file_size(info.new_file) == biggest_written); } SECTION("write to new file") { @@ -879,7 +879,7 @@ TEST_CASE("BatchedWriteRSRangeLarge", } test::test_fclose(); REQUIRE(test::status_orig == 0); - REQUIRE(fs::file_size(info.new_file) == total_written); + REQUIRE(stdfs::file_size(info.new_file) == total_written); } posttest(); } diff --git a/adapter/test/stdio/stdio_adapter_test.cpp b/adapter/test/stdio/stdio_adapter_test.cpp index e27516b57..37e337dc0 100644 --- a/adapter/test/stdio/stdio_adapter_test.cpp +++ b/adapter/test/stdio/stdio_adapter_test.cpp @@ -20,12 +20,12 @@ #include "catch_config.h" #include "adapter_test_utils.h" #if HERMES_INTERCEPT == 1 -#include "stdio/stdio.h" +#include "stdio/real_api.h" #endif #include "adapter_test_utils.h" -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; namespace hermes::adapter::stdio::test { struct Arguments { @@ -71,7 +71,7 @@ int finalize() { } int pretest() { - fs::path fullpath = args.directory; + stdfs::path fullpath = args.directory; fullpath /= args.filename; info.new_file = fullpath.string() + "_new_" + std::to_string(getpid()); info.existing_file = fullpath.string() + "_ext_" + std::to_string(getpid()); @@ -79,25 +79,27 @@ int pretest() { std::to_string(getpid()); info.existing_file_cmp = fullpath.string() + "_ext_cmp" + "_" + std::to_string(getpid()); - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); - if (!fs::exists(info.existing_file)) { + if (stdfs::exists(info.new_file)) stdfs::remove(info.new_file); + if (stdfs::exists(info.new_file_cmp)) stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); + if (!stdfs::exists(info.existing_file)) { std::string cmd = "{ tr -dc '[:alnum:]' < /dev/urandom | head -c " + std::to_string(args.request_size * info.num_iterations) + "; } > " + info.existing_file + " 2> /dev/null"; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file) == + REQUIRE(stdfs::file_size(info.existing_file) == args.request_size * info.num_iterations); - info.total_size = fs::file_size(info.existing_file); + info.total_size = stdfs::file_size(info.existing_file); } - if (!fs::exists(info.existing_file_cmp)) { + if (!stdfs::exists(info.existing_file_cmp)) { std::string cmd = "cp " + info.existing_file + " " + info.existing_file_cmp; int status = system(cmd.c_str()); REQUIRE(status != -1); - REQUIRE(fs::file_size(info.existing_file_cmp) == + REQUIRE(stdfs::file_size(info.existing_file_cmp) == args.request_size * info.num_iterations); } REQUIRE(info.total_size > 0); @@ -113,10 +115,10 @@ int posttest(bool compare_data = true) { INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.existing_file); INTERCEPTOR_LIST->hermes_flush_exclusion.insert(info.new_file); #endif - if (compare_data && fs::exists(info.new_file) && - fs::exists(info.new_file_cmp)) { - size_t size = fs::file_size(info.new_file); - REQUIRE(size == fs::file_size(info.new_file_cmp)); + if (compare_data && stdfs::exists(info.new_file) && + stdfs::exists(info.new_file_cmp)) { + size_t size = stdfs::file_size(info.new_file); + REQUIRE(size == stdfs::file_size(info.new_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -144,11 +146,11 @@ int posttest(bool compare_data = true) { REQUIRE(char_mismatch == 0); } } - if (compare_data && fs::exists(info.existing_file) && - fs::exists(info.existing_file_cmp)) { - size_t size = fs::file_size(info.existing_file); - if (size != fs::file_size(info.existing_file_cmp)) sleep(1); - REQUIRE(size == fs::file_size(info.existing_file_cmp)); + if (compare_data && stdfs::exists(info.existing_file) && + stdfs::exists(info.existing_file_cmp)) { + size_t size = stdfs::file_size(info.existing_file); + if (size != stdfs::file_size(info.existing_file_cmp)) sleep(1); + REQUIRE(size == stdfs::file_size(info.existing_file_cmp)); if (size > 0) { std::vector d1(size, '0'); std::vector d2(size, '1'); @@ -174,10 +176,14 @@ int posttest(bool compare_data = true) { } } /* Clean up. */ - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); #if HERMES_INTERCEPT == 1 INTERCEPTOR_LIST->hermes_flush_exclusion.erase(info.existing_file_cmp); diff --git a/adapter/test/vfd/hermes_vfd_basic_test.cc b/adapter/test/vfd/hermes_vfd_basic_test.cc index 4ab25aa08..67f8b9277 100644 --- a/adapter/test/vfd/hermes_vfd_basic_test.cc +++ b/adapter/test/vfd/hermes_vfd_basic_test.cc @@ -710,7 +710,7 @@ TEST_CASE("ScratchMode", "[scratch]") { REQUIRE(test::hermes_herr >= 0); if (info.scratch_mode) { - REQUIRE(!fs::exists(info.new_file)); + REQUIRE(!stdfs::exists(info.new_file)); } } diff --git a/adapter/test/vfd/hermes_vfd_test.cc b/adapter/test/vfd/hermes_vfd_test.cc index e557c8e6d..796835879 100644 --- a/adapter/test/vfd/hermes_vfd_test.cc +++ b/adapter/test/vfd/hermes_vfd_test.cc @@ -26,7 +26,7 @@ #include "adapter_test_utils.h" #include "catch_config.h" -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; using hermes::f32; using hermes::u32; @@ -372,7 +372,7 @@ int init(int* argc, char*** argv) { info.read_data[i] = 0.0f; } - fs::path fullpath = args.directory; + stdfs::path fullpath = args.directory; fullpath /= args.filename; std::string suffix = std::to_string(getpid()) + info.hdf5_extension; info.new_file = fullpath.string() + "_new_" + suffix; @@ -404,10 +404,14 @@ int finalize() { * Remove all files generated by the tests */ void CleanupFiles() { - if (fs::exists(info.new_file)) fs::remove(info.new_file); - if (fs::exists(info.new_file_cmp)) fs::remove(info.new_file_cmp); - if (fs::exists(info.existing_file)) fs::remove(info.existing_file); - if (fs::exists(info.existing_file_cmp)) fs::remove(info.existing_file_cmp); + if (stdfs::exists(info.new_file)) + stdfs::remove(info.new_file); + if (stdfs::exists(info.new_file_cmp)) + stdfs::remove(info.new_file_cmp); + if (stdfs::exists(info.existing_file)) + stdfs::remove(info.existing_file); + if (stdfs::exists(info.existing_file_cmp)) + stdfs::remove(info.existing_file_cmp); } /** @@ -419,7 +423,7 @@ int Pretest() { CleanupFiles(); GenHdf5File(info.existing_file, info.nelems_per_dataset, info.num_iterations); - info.total_size = fs::file_size(info.existing_file); + info.total_size = stdfs::file_size(info.existing_file); std::string cmd = "cp " + info.existing_file + " " + info.existing_file_cmp; int status = system(cmd.c_str()); REQUIRE(status != -1); @@ -433,7 +437,7 @@ int Pretest() { * POSIX VFD are the same. */ void CheckResults(const std::string &file1, const std::string &file2) { - if (fs::exists(file1) && fs::exists(file2)) { + if (stdfs::exists(file1) && stdfs::exists(file2)) { std::string h5diff_cmd = "h5diff " + file1 + " " + file2; int status = system(h5diff_cmd.c_str()); if (status != 0) { diff --git a/data_stager/CMakeLists.txt b/data_stager/CMakeLists.txt new file mode 100644 index 000000000..7e754dad0 --- /dev/null +++ b/data_stager/CMakeLists.txt @@ -0,0 +1,47 @@ +include_directories(${CMAKE_SOURCE_DIR}/adapter ${CMAKE_CURRENT_SOURCE_DIR}) +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DHERMES_RPC_THALLIUM") + +add_library(hermes_data_stager + ${CMAKE_CURRENT_SOURCE_DIR}/stagers/unix_stager.cc) +add_dependencies(hermes_data_stager hermes_posix_backend) +target_link_libraries(hermes_data_stager hermes_posix_backend) + +add_executable(stage_in ${CMAKE_CURRENT_SOURCE_DIR}/stage_in.cc) +add_dependencies(stage_in hermes_data_stager) +target_link_libraries(stage_in hermes_data_stager) + +add_executable(stage_out ${CMAKE_CURRENT_SOURCE_DIR}/stage_out.cc) +add_dependencies(stage_out hermes_data_stager) +target_link_libraries(stage_out hermes_data_stager) + +add_executable(unix_stage_test test/unix/hermes_staging_unix_test.cc) +add_dependencies(unix_stage_test hermes_data_stager) +target_link_libraries(unix_stage_test hermes_data_stager) + +install( + TARGETS + hermes_data_stager + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} +) +install( + TARGETS + stage_in + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} +) +install( + TARGETS + stage_out + EXPORT + ${HERMES_EXPORTED_TARGETS} + LIBRARY DESTINATION ${HERMES_INSTALL_LIB_DIR} + ARCHIVE DESTINATION ${HERMES_INSTALL_LIB_DIR} + RUNTIME DESTINATION ${HERMES_INSTALL_BIN_DIR} +) \ No newline at end of file diff --git a/data_stager/data_stager.h b/data_stager/data_stager.h new file mode 100644 index 000000000..759520bca --- /dev/null +++ b/data_stager/data_stager.h @@ -0,0 +1,65 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_DATA_STAGER_STAGE_IN_H_ +#define HERMES_DATA_STAGER_STAGE_IN_H_ + +#include "posix/fs_api.h" +#include "hermes_types.h" +#include + +namespace hermes { + +enum class DataStagerType { + kUnix, + kHdf5 +}; + +class DataStagerTypeConv { + public: + static DataStagerType from_url(const std::string &url) { + if (url.rfind("h5::", 0) != std::string::npos) { + return DataStagerType::kHdf5; + } else { + return DataStagerType::kUnix; + } + } +}; + +class DataStager { + public: + virtual void StageIn(std::string url, PlacementPolicy dpe) = 0; + virtual void StageIn(std::string url, + off_t off, size_t size, PlacementPolicy dpe) = 0; + virtual void StageOut(std::string url) = 0; + + protected: + void DivideRange(off_t off, size_t size, + off_t &new_off, size_t &new_size) { + auto mdm = Singleton::GetInstance(); + int concurrency = 1; + int rank = 0; + if (mdm->is_mpi) { + concurrency = mdm->comm_size; + rank = mdm->rank; + } + new_size = size / concurrency; + new_off = off + new_size * rank; + if (rank == concurrency - 1) { + new_size += size % concurrency; + } + } +}; + +} // namespace hermes + +#endif // HERMES_DATA_STAGER_STAGE_IN_H_ diff --git a/data_stager/data_stager_factory.h b/data_stager/data_stager_factory.h new file mode 100644 index 000000000..22932e155 --- /dev/null +++ b/data_stager/data_stager_factory.h @@ -0,0 +1,41 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_DATA_STAGER_STAGE_FACTORY_H_ +#define HERMES_DATA_STAGER_STAGE_FACTORY_H_ + +#include "data_stager.h" +#include "stagers/unix_stager.h" + +namespace hermes { + +class DataStagerFactory { + public: + static std::unique_ptr Get(const std::string& url) { + return Get(DataStagerTypeConv::from_url(url)); + } + + static std::unique_ptr Get(DataStagerType stager) { + switch (stager) { + case DataStagerType::kUnix: { + return std::make_unique(); + } + default: { + return nullptr; + } + } + } +}; + +} // namespace hermes + +#endif // HERMES_DATA_STAGER_STAGE_FACTORY_H_ diff --git a/data_stager/stage_in.cc b/data_stager/stage_in.cc new file mode 100644 index 000000000..226142c90 --- /dev/null +++ b/data_stager/stage_in.cc @@ -0,0 +1,43 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include "posix/fs_api.h" +#include "hermes_types.h" +#include +#include "data_stager_factory.h" + +using hermes::api::PlacementPolicyConv; +using hermes::api::PlacementPolicy; +using hermes::DataStager; +using hermes::DataStagerFactory; + +int main(int argc, char **argv) { + if (argc != 4) { + std::cout << "Usage: mpirun -n [nprocs]" << + " ./stage_in [url] [offset] [size] [dpe]" << std::endl; + exit(1); + } + auto mdm = Singleton::GetInstance(); + MPI_Init(&argc, &argv); + mdm->InitializeHermes(true); + off_t off; + size_t size; + std::string url = argv[1]; + std::stringstream(argv[2]) >> off; + std::stringstream(argv[3]) >> size; + PlacementPolicy dpe = PlacementPolicyConv::to_enum(argv[4]); + auto stager = DataStagerFactory::Get(url); + stager->StageIn(url, dpe); + mdm->FinalizeHermes(); + MPI_Finalize(); +} diff --git a/data_stager/stage_out.cc b/data_stager/stage_out.cc new file mode 100644 index 000000000..e8c9d0822 --- /dev/null +++ b/data_stager/stage_out.cc @@ -0,0 +1,35 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include +#include "posix/fs_api.h" +#include "hermes_types.h" +#include +#include "data_stager_factory.h" + +using hermes::api::PlacementPolicyConv; +using hermes::api::PlacementPolicy; +using hermes::DataStager; +using hermes::DataStagerFactory; + +int main(int argc, char **argv) { + if (argc != 1) { + std::cout << "Usage: ./stage_out [url]" << std::endl; + exit(1); + } + auto mdm = Singleton::GetInstance(); + mdm->InitializeHermes(false); + std::string url = argv[1]; + auto stager = DataStagerFactory::Get(url); + stager->StageOut(url); + mdm->FinalizeHermes(); +} diff --git a/data_stager/stagers/unix_stager.cc b/data_stager/stagers/unix_stager.cc new file mode 100644 index 000000000..e383956f8 --- /dev/null +++ b/data_stager/stagers/unix_stager.cc @@ -0,0 +1,99 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "unix_stager.h" + +using hermes::adapter::posix::PosixFS; +using hermes::api::PlacementPolicyConv; +using hermes::api::PlacementPolicy; +using hermes::adapter::fs::IoOptions; +namespace stdfs = std::experimental::filesystem; + +namespace hermes { + +void UnixStager::StageIn(std::string path, PlacementPolicy dpe) { + if (stdfs::is_regular_file(path)) { + FileStageIn(path, dpe); + } else if (stdfs::is_directory(path)) { + DirectoryStageIn(path, dpe); + } else { + LOG(ERROR) << "Unix stage in is neither a file or directory" << std::endl; + } +} + +void UnixStager::FileStageIn(std::string path, PlacementPolicy dpe) { + off_t per_proc_off; + size_t per_proc_size; + DivideRange(0, stdfs::file_size(path), per_proc_off, per_proc_size); + FileStageIn(path, per_proc_off, per_proc_size, dpe); +} + +void UnixStager::DirectoryStageIn(std::string path, PlacementPolicy dpe) { + for (auto &file_path : stdfs::directory_iterator(path)) { + FileStageIn(file_path.path(), dpe); + } +} + +void UnixStager::StageIn(std::string path, off_t off, + size_t size, PlacementPolicy dpe) { + if (stdfs::is_regular_file(path)) { + FileStageIn(path, off, size, dpe); + } else if (stdfs::is_directory(path)) { + LOG(ERROR) << "Unix stage-in with offset " << + "is not supported for directories" << std::endl; + } else { + LOG(ERROR) << "Unix stage-in is neither a file or directory" << std::endl; + } +} + +void UnixStager::FileStageIn(std::string path, + off_t off, size_t size, PlacementPolicy dpe) { + auto fs_api = PosixFS(); + std::vector buf(size); + AdapterStat stat; + bool stat_exists; + IoStatus io_status; + File f = fs_api.Open(stat, path); + fs_api.Read(f, stat, buf.data(), off, size, + io_status, IoOptions::WithParallelDpe(dpe)); + fs_api.Close(f, stat_exists, false); +} + +void UnixStager::StageOut(std::string path) { + if (stdfs::is_regular_file(path)) { + FileStageOut(path); + } else if (stdfs::is_directory(path)) { + DirectoryStageOut(path); + } else { + LOG(ERROR) << "Unix stage-out is neither a file or directory" << std::endl; + } +} + +void UnixStager::FileStageOut(std::string path) { + auto fs_api = PosixFS(); + AdapterStat stat; + bool stat_exists; + File f = fs_api.Open(stat, path); + if (!f.status_) { + LOG(INFO) << "Couldn't open file: " << path << std::endl; + return; + } + fs_api.Close(f, stat_exists, false); +} + +void UnixStager::DirectoryStageOut(std::string path) { + for (auto &file_path : stdfs::directory_iterator(path)) { + FileStageOut(file_path.path()); + } +} + +} // namespace hermes diff --git a/data_stager/stagers/unix_stager.h b/data_stager/stagers/unix_stager.h new file mode 100644 index 000000000..ab28c1d25 --- /dev/null +++ b/data_stager/stagers/unix_stager.h @@ -0,0 +1,39 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef HERMES_DATA_STAGER_STAGERS_UNIX_STAGE_H_ +#define HERMES_DATA_STAGER_STAGERS_UNIX_STAGE_H_ + +#include "posix/fs_api.h" +#include "../data_stager.h" + +namespace hermes { + +class UnixStager : public DataStager { + public: + void StageIn(std::string url, PlacementPolicy dpe) override; + void FileStageIn(std::string path, PlacementPolicy dpe); + void DirectoryStageIn(std::string path, PlacementPolicy dpe); + + void StageIn(std::string url, + off_t off, size_t size, PlacementPolicy dpe) override; + void FileStageIn(std::string path, + off_t off, size_t size, PlacementPolicy dpe); + + void StageOut(std::string url) override; + void FileStageOut(std::string path); + void DirectoryStageOut(std::string path); +}; + +} // namespace hermes + +#endif // HERMES_DATA_STAGER_STAGERS_UNIX_STAGE_H_ diff --git a/data_stager/test/unix/hermes_staging_unix_test.cc b/data_stager/test/unix/hermes_staging_unix_test.cc new file mode 100644 index 000000000..0e9316467 --- /dev/null +++ b/data_stager/test/unix/hermes_staging_unix_test.cc @@ -0,0 +1,151 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +* Distributed under BSD 3-Clause license. * +* Copyright by The HDF Group. * +* Copyright by the Illinois Institute of Technology. * +* All rights reserved. * +* * +* This file is part of Hermes. The full Hermes copyright notice, including * +* terms governing use, modification, and redistribution, is contained in * +* the COPYING file, which can be found at the top directory. If you do not * +* have access to the file, you may request a copy from help@hdfgroup.org. * +* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "../../stagers/unix_stager.h" +#include + +using hermes::UnixStager; +using hermes::PlacementPolicy; +using hermes::adapter::posix::PosixFS; +namespace stdfs = std::experimental::filesystem; + +void CreateFile(const std::string &path, int nonce, size_t size) { + size_t count = size / sizeof(int); + auto mdm = Singleton::GetInstance(); + int rank = 0; + if (mdm->is_mpi) { + rank = mdm->rank; + } + if (rank == 0) { + FILE *file = fopen(path.c_str(), "w"); + std::vector data(count, nonce); + fwrite(data.data(), 1, size, file); + fclose(file); + } + if (mdm->is_mpi) { + MPI_Barrier(MPI_COMM_WORLD); + } +} + +void VerifyFile(const std::string &path, int nonce, size_t size) { + auto mdm = Singleton::GetInstance(); + if (mdm->rank != 0) { + return; + } + size_t count = size / sizeof(int); + + if (stdfs::file_size(path) != size) { + std::cout << "File sizes are not equal" << std::endl; + exit(1); + } + + FILE *file = fopen(path.c_str(), "r"); + std::vector data(count); + fread(data.data(), 1, size, file); + for (size_t i = 0; i < count ; ++i) { + if (data[i] != nonce) { + std::cout << "File verification failed" + << " at offset: " << i << "/" << count << std::endl; + exit(1); + } + } + fclose(file); +} + +void test_file_stage() { + UnixStager stager; + auto mdm = Singleton::GetInstance(); + std::string path = "/tmp/test.txt"; + int nonce = mdm->rank + 122; + int new_nonce = ~nonce; + size_t file_size = MEGABYTES(16); + PlacementPolicy dpe = PlacementPolicy::kRoundRobin; + + // Create a 16MB file + CreateFile(path, nonce, file_size); + VerifyFile(path, nonce, file_size); + + // Stage in the 16MB file + stager.StageIn(path, PlacementPolicy::kRoundRobin); + if (mdm->is_mpi) { MPI_Barrier(MPI_COMM_WORLD); } + + // Ensure the bucket corresponding to the file name is created + bool bucket_exists = mdm->GetHermes()->BucketExists(path); + if (!bucket_exists) { + std::cout << "Stage in failed to load bucket" << std::endl; + exit(1); + } + auto bkt = std::make_shared(path, mdm->GetHermes()); + + // Verify the bucket contents + if (mdm->rank == 0) { + for (int i = 0; i < 16; ++i) { + // A blob should be created for each 1MB of data + // This is because the hermes file page size (kPageSize) is 1MB + hermes::adapter::BlobPlacement p; + p.page_ = i; + std::string blob_name = p.CreateBlobName(); + bool has_blob = bkt->ContainsBlob(blob_name); + if (!has_blob) { + std::cout << "Stage in failed to load blob: " << i + << " in file: " << path << std::endl; + exit(1); + } + + // Get blob size and contents + hapi::Blob blob(0); + auto size = bkt->Get(blob_name, blob); + blob.resize(size); + bkt->Get(blob_name, blob); + auto count = size / sizeof(int); + int *blob_data = (int *)blob.data(); + + // Ensure the blob contents are correct + for (size_t off = 0; off < count; ++off) { + if (blob_data[off] != nonce) { + std::cout << "Stage-in has improper content " << blob_data[off] + << " != " << nonce << " in file: " << path << std::endl; + exit(1); + } + } + } + + // Override the original blob contents with new contents + auto count = file_size / sizeof(int); + std::vector new_data(count, new_nonce); + bool stat_exists; + AdapterStat stat; + PosixFS fs_api; + IoStatus io_status; + File f = fs_api.Open(stat, path); + fs_api.Write(f, stat, new_data.data(), 0, file_size, io_status, + IoOptions::WithParallelDpe(dpe)); + fs_api.Close(f, stat_exists, false); + } + + // Stage out the new file (sync its contents) + MPI_Barrier(MPI_COMM_WORLD); + stager.StageOut(path); + + // Verify the contents have been written + VerifyFile(path, new_nonce, file_size); +} + +int main(int argc, char **argv) { + MPI_Init(&argc, &argv); + auto mdm = Singleton::GetInstance(); + mdm->InitializeHermes(true); + test_file_stage(); + mdm->FinalizeHermes(); + MPI_Finalize(); + std::cout << "Evaluation passed!" << std::endl; +} diff --git a/src/data_placement_engine_factory.h b/src/data_placement_engine_factory.h index 8ceced833..3ef14afb2 100644 --- a/src/data_placement_engine_factory.h +++ b/src/data_placement_engine_factory.h @@ -40,6 +40,7 @@ class DPEFactory { case PlacementPolicy::kMinimizeIoTime: { return std::make_unique(); } + case PlacementPolicy::kNone: default: { // TODO(luke): @errorhandling not implemented LOG(FATAL) << "PlacementPolicy not implemented" << std::endl; diff --git a/src/dpe/minimize_io_time.cc b/src/dpe/minimize_io_time.cc index 917697518..f88fc0fde 100644 --- a/src/dpe/minimize_io_time.cc +++ b/src/dpe/minimize_io_time.cc @@ -65,8 +65,7 @@ Status MinimizeIoTime::Placement(const std::vector &blob_sizes, // Constraint #2: Capacity constraints for (size_t j = 0; j < num_targets; ++j) { double rem_cap_thresh = - static_cast(node_state[j]) - - static_cast(node_state[j]) * minimum_remaining_capacity; + static_cast(node_state[j]) * (1 - minimum_remaining_capacity); double est_rem_cap = capacity_change_threshold * node_state[j]; double max_capacity = std::max({rem_cap_thresh, est_rem_cap}); if (max_capacity > 0) { diff --git a/src/hermes_types.h b/src/hermes_types.h index 546a57528..e42584101 100644 --- a/src/hermes_types.h +++ b/src/hermes_types.h @@ -72,6 +72,7 @@ enum class PlacementPolicy { kRandom, /**< Random blob placement */ kRoundRobin, /**< Round-Robin (around devices) blob placement */ kMinimizeIoTime, /**< LP-based blob placement, minimize I/O time */ + kNone, /** No DPE for cases we want it disabled */ }; class PlacementPolicyConv { @@ -87,9 +88,25 @@ class PlacementPolicyConv { case PlacementPolicy::kMinimizeIoTime: { return "PlacementPolicy::kMinimizeIoTime"; } + case PlacementPolicy::kNone: { + return "PlacementPolicy::kNone"; + } } return "PlacementPolicy::Invalid"; } + + static PlacementPolicy to_enum(const std::string &policy) { + if (policy.find("kRandom") != std::string::npos) { + return PlacementPolicy::kRandom; + } else if (policy.find("kRoundRobin") != std::string::npos) { + return PlacementPolicy::kRoundRobin; + } else if (policy.find("kMinimizeIoTime") != std::string::npos) { + return PlacementPolicy::kMinimizeIoTime; + } else if (policy.find("kNone") != std::string::npos) { + return PlacementPolicy::kNone; + } + return PlacementPolicy::kNone; + } }; diff --git a/src/memory_management.cc b/src/memory_management.cc index 3d367d344..76bccfa3d 100644 --- a/src/memory_management.cc +++ b/src/memory_management.cc @@ -471,22 +471,6 @@ Ticket TryBeginTicketMutex(TicketMutex *mutex, Ticket *existing_ticket) { return result; } -/** - * - */ -bool BeginTicketMutexIfNoWait(TicketMutex *mutex) { - u32 serving = mutex->serving.load(); - u32 ticket = mutex->ticket.load(); - u32 next = ticket + 1; - - bool result = false; - if (serving == ticket) { - result = mutex->ticket.compare_exchange_strong(ticket, next); - } - - return result; -} - void BeginTicketMutex(TicketMutex *mutex) { u32 ticket = mutex->ticket.fetch_add(1); while (ticket != mutex->serving.load()) { diff --git a/src/memory_management.h b/src/memory_management.h index a629dd179..5f534043d 100644 --- a/src/memory_management.h +++ b/src/memory_management.h @@ -41,6 +41,7 @@ struct TicketMutex { struct Ticket { u32 ticket; bool acquired; + Ticket() : ticket(-1), acquired(false) {} }; struct RwLock { diff --git a/src/metadata_management.cc b/src/metadata_management.cc index ff3283f4c..583e2b0b9 100644 --- a/src/metadata_management.cc +++ b/src/metadata_management.cc @@ -1543,7 +1543,6 @@ bool LocalLockBlob(SharedMemoryContext *context, BlobID blob_id) { } ticket = &t; } - return result; } @@ -1562,6 +1561,13 @@ bool LocalUnlockBlob(SharedMemoryContext *context, BlobID blob_id) { return result; } +void GetPidTid(u32 &pid, u32 &tid) { + pid = getpid(); + ABT_unit_id tid_argo; + ABT_thread_self_id(&tid_argo); + tid = tid_argo; +} + bool LockBlob(SharedMemoryContext *context, RpcContext *rpc, BlobID blob_id) { u32 target_node = GetBlobNodeId(blob_id); bool result = false; @@ -1570,7 +1576,6 @@ bool LockBlob(SharedMemoryContext *context, RpcContext *rpc, BlobID blob_id) { } else { result = RpcCall(rpc, target_node, "RemoteLockBlob", blob_id); } - return result; } diff --git a/test/trait_test.cc b/test/trait_test.cc index 67d08f8a3..ff99351d8 100644 --- a/test/trait_test.cc +++ b/test/trait_test.cc @@ -19,7 +19,7 @@ #include "vbucket.h" #include "test_utils.h" -namespace fs = std::experimental::filesystem; +namespace stdfs = std::experimental::filesystem; namespace hapi = hermes::api; struct Arguments { @@ -94,7 +94,7 @@ TEST_CASE("CustomTrait", using HermesPtr = std::shared_ptr; HermesPtr hermes_app = hapi::InitHermes(args.config.c_str(), true); - fs::path fullpath = args.directory; + stdfs::path fullpath = args.directory; fullpath /= args.filename; std::string fullpath_str = fullpath.string(); @@ -125,10 +125,10 @@ TEST_CASE("CustomTrait", file_vbucket.Attach(&persist_trait); file_vbucket.Destroy(); file_bucket.Destroy(); - REQUIRE(fs::exists(fullpath_str)); + REQUIRE(stdfs::exists(fullpath_str)); size_t total_bytes = args.iterations * args.request_size; - REQUIRE(fs::file_size(fullpath_str) == total_bytes); + REQUIRE(stdfs::file_size(fullpath_str) == total_bytes); auto read_blob = hapi::Blob(total_bytes, '0'); FILE* fh = fopen(fullpath_str.c_str(), "r+"); REQUIRE(fh != nullptr);