diff --git a/CMakeLists.txt b/CMakeLists.txt index d98cd35a..f06a9f29 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -139,7 +139,7 @@ endif () # -------------------------------------------------------------------------------- # # -------------------------------------------------------------------------------- # -# Executables +# Set up for building executables # -------------------------------------------------------------------------------- # # Requirements for GCC @@ -169,12 +169,8 @@ find_package(Threads REQUIRED) # ---------- filesystem ---------- # -include(include_cxx_filesystem_library) -include_cxx_filesystem_library() - -# Xcode 11 Beta Release Notes -# Clang now supports the C++17 library for iOS 13, macOS 10.15, watchOS 6, and tvOS 13. (50988273) -# https://developer.apple.com/documentation/xcode_release_notes/xcode_11_beta_release_notes?language=objc +include(check_cxx_filesystem_library) +check_cxx_filesystem_library() # ---------- UMap ---------- # @@ -198,7 +194,7 @@ set(Boost_NO_BOOST_CMAKE ON) find_package(Boost 1.64 QUIET) if (NOT Boost_FOUND) FetchContent_Declare(Boost - URL https://boostorg.jfrog.io/artifactory/main/release/1.78.0/source/boost_1_78_0.tar.bz2) + URL https://boostorg.jfrog.io/artifactory/main/release/1.83.0/source/boost_1_83_0.tar.bz2) FetchContent_GetProperties(Boost) if (NOT Boost_POPULATED) FetchContent_Populate(Boost) @@ -258,13 +254,11 @@ function(common_setup_for_metall_executable name) # -------------------- # ----- CXX17 Filesystem Lib----- # - # include_cxx_filesystem_library module must be executed first - if (FOUND_CXX17_FILESYSTEM_LIB) - if (REQUIRE_LIB_STDCXX_FS) + # GNU compilers prior to 9.1 requires linking with stdc++fs + if (("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU") OR ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")) + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.1) target_link_libraries(${name} PRIVATE stdc++fs) endif () - elseif () - target_compile_definitions(${name} PRIVATE "METALL_DISABLE_CXX17_FILESYSTEM_LIB") endif () # -------------------- diff --git a/bench/bfs/run_bfs_bench_metall.cpp b/bench/bfs/run_bfs_bench_metall.cpp index ccf3b0b4..a4587060 100644 --- a/bench/bfs/run_bfs_bench_metall.cpp +++ b/bench/bfs/run_bfs_bench_metall.cpp @@ -29,7 +29,7 @@ int main(int argc, char *argv[]) { // metall::logger::set_log_level(metall::logger::level::verbose); metall::manager manager(metall::open_read_only, - option.graph_file_name_list[0].c_str()); + option.graph_file_name_list[0]); auto adj_list = manager.find(option.graph_key_name.c_str()).first; diff --git a/bench/bfs/run_bfs_bench_metall_multiple.cpp b/bench/bfs/run_bfs_bench_metall_multiple.cpp index 6fa7e2d3..fd27f6a2 100644 --- a/bench/bfs/run_bfs_bench_metall_multiple.cpp +++ b/bench/bfs/run_bfs_bench_metall_multiple.cpp @@ -33,7 +33,7 @@ int main(int argc, char *argv[]) { std::vector managers; for (const auto &file_name : option.graph_file_name_list) { managers.emplace_back( - new metall::manager(metall::open_read_only, file_name.c_str())); + new metall::manager(metall::open_read_only, file_name)); } auto adj_list = adjacency_list_type(option.graph_key_name, managers.begin(), diff --git a/bench/simple_alloc/run_simple_allocation_bench_metall.cpp b/bench/simple_alloc/run_simple_allocation_bench_metall.cpp index 1b9d5249..ff3c0246 100644 --- a/bench/simple_alloc/run_simple_allocation_bench_metall.cpp +++ b/bench/simple_alloc/run_simple_allocation_bench_metall.cpp @@ -13,7 +13,7 @@ int main(int argc, char *argv[]) { const auto option = simple_alloc_bench::parse_option(argc, argv); { - metall::manager manager(metall::create_only, option.datastore_path.c_str()); + metall::manager manager(metall::create_only, option.datastore_path); simple_alloc_bench::run_bench(option, manager.get_allocator()); } metall::manager::remove(option.datastore_path.c_str()); diff --git a/cmake/check_cxx_filesystem_library.cmake b/cmake/check_cxx_filesystem_library.cmake new file mode 100644 index 00000000..88eb5ad7 --- /dev/null +++ b/cmake/check_cxx_filesystem_library.cmake @@ -0,0 +1,25 @@ +# Checks if the C++17 library is available. +function(check_cxx_filesystem_library) + + set(FOUND_CXX17_FILESYSTEM_LIB FALSE PARENT_SCOPE) + set(REQUIRE_LIB_STDCXX_FS FALSE PARENT_SCOPE) + + # Check if C++17 header files are available + # If not, uses our own implementation. + include(CheckIncludeFileCXX) + CHECK_INCLUDE_FILE_CXX(filesystem FOUND_FILESYSTEM_HEADER) + if (NOT FOUND_FILESYSTEM_HEADER) + message(FATAL_ERROR "Cannot find the C++17 library.") + endif () + + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # Clang or AppleClang + if (${CMAKE_HOST_SYSTEM_NAME} MATCHES "Darwin") # macOS + include(get_macos_version) + get_macos_version() # Get macOS version + message(VERBOSE "Detected macOS version ${MACOS_VERSION}") + if (MACOS_VERSION VERSION_LESS 10.15) # macOS < 10.15 + message(FATAL_ERROR "macOS >= 10.15 is required to use the C++17 library.") + endif () + endif () + endif () +endfunction() \ No newline at end of file diff --git a/cmake/include_cxx_filesystem_library.cmake b/cmake/include_cxx_filesystem_library.cmake deleted file mode 100644 index a54ca061..00000000 --- a/cmake/include_cxx_filesystem_library.cmake +++ /dev/null @@ -1,40 +0,0 @@ -# Find the correct option to link the C++17 library. -# If it is not available, uses own implementation. -function(include_cxx_filesystem_library) - - set(FOUND_CXX17_FILESYSTEM_LIB TRUE PARENT_SCOPE) - set(REQUIRE_LIB_STDCXX_FS FALSE PARENT_SCOPE) - - # Check if C++17 header files are available - # If not, uses our own implementation. - include(CheckIncludeFileCXX) - CHECK_INCLUDE_FILE_CXX(filesystem FOUND_FILESYSTEM_HEADER) - if (NOT FOUND_FILESYSTEM_HEADER) - message(STATUS "Cannot find the C++17 library.") - set(FOUND_CXX17_FILESYSTEM_LIB FALSE PARENT_SCOPE) - return() - endif () - - # Find the correct option to link the C++17 library - # GCC - # Any platform: stdc++fs - # LLVM - # Assumes LLVM >= 9.0 and libc++ >= 7.0. - # Clang + macOS >= 10.15: nothing special is required to use - # Clang + macOS < 10.15: uses our own implementation - # Clang + the other OS: nothing special is required to use - if (("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")) # GCC - set(REQUIRE_LIB_STDCXX_FS TRUE PARENT_SCOPE) - elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # Clang or AppleClang - if (${CMAKE_HOST_SYSTEM_NAME} MATCHES "Darwin") # macOS - include(get_macos_version) - get_macos_version() # Get macOS version - message(VERBOSE "macOS version ${MACOS_VERSION}") - if (MACOS_VERSION VERSION_LESS 10.15) # macOS < 10.15 - message(STATUS "macOS >= 10.15 is required to use the C++17 library.") - set(FOUND_CXX17_FILESYSTEM_LIB FALSE PARENT_SCOPE) - endif () - endif () - endif () - -endfunction() \ No newline at end of file diff --git a/docs/readthedocs/basics/getting_started.md b/docs/readthedocs/basics/getting_started.md index 45965228..79c8a629 100644 --- a/docs/readthedocs/basics/getting_started.md +++ b/docs/readthedocs/basics/getting_started.md @@ -69,22 +69,11 @@ g++ -std=c++17 your_program.cpp -lstdc++fs -I${BOOST_ROOT}/include -I${METALL_RO Clang (or Apple clang) could be used instead of GCC to build Metall. However, we haven't tested it intensively. +To run on macOS, Metall requires macOS >= 10.15. + Also, Boost C++ Libraries 1.69 or more may be required if one wants to build Metall with Clang + CUDA. -**On macOS >= 10.15 or Linux** - ```bash -# Remove "-lstdc++fs" option clang++ -std=c++17 [tutorial_program.cpp] -I../../include -I${BOOST_ROOT} -``` - -**On macOS < 10.15** - -The C++17 library is not available on macOS < 10.15. -One has to stop using C++17 library in Metall. -If METALL_DISABLE_CXX17_FILESYSTEM_LIB macro is defined, Metall uses its own file system operation implementation. - -```bash -clang++ -std=c++17 [tutorial_program.cpp] -I../../include -I${BOOST_ROOT} -DMETALL_DISABLE_CXX17_FILESYSTEM_LIB -``` +``` \ No newline at end of file diff --git a/include/metall/detail/file.hpp b/include/metall/detail/file.hpp index e987d38c..d351d035 100644 --- a/include/metall/detail/file.hpp +++ b/include/metall/detail/file.hpp @@ -28,37 +28,15 @@ #include #include #include - -#ifdef __has_include - -// Check if the Filesystem library is available or disabled by the user -#if __has_include() && !defined(METALL_DISABLE_CXX17_FILESYSTEM_LIB) #include -#else -#ifdef METALL_VERBOSE_SYSTEM_SUPPORT_WARNING -#warning "The Filesystem library is not available or disabled by the user." -#endif -#endif - -#else // __has_include is not defined - -#ifdef METALL_VERBOSE_SYSTEM_SUPPORT_WARNING -#warning \ - "__has_include is not defined, consequently disable the Filesystem library." -#endif // METALL_VERBOSE_SYSTEM_SUPPORT_WARNING - -#endif // #ifdef __has_include #include namespace metall::mtlldetail { -#if defined(__cpp_lib_filesystem) && \ - !defined(METALL_DISABLE_CXX17_FILESYSTEM_LIB) namespace { namespace fs = std::filesystem; } -#endif inline bool os_close(const int fd) { if (::close(fd) == -1) { @@ -76,7 +54,7 @@ inline bool os_fsync(const int fd) { return true; } -inline bool fsync(const std::string &path) { +inline bool fsync(const fs::path &path) { const int fd = ::open(path.c_str(), O_RDONLY); if (fd == -1) { logger::perror(logger::level::error, __FILE__, __LINE__, "open"); @@ -90,9 +68,7 @@ inline bool fsync(const std::string &path) { return ret; } -inline bool fsync_recursive(const std::string &path) { -#if defined(__cpp_lib_filesystem) && \ - !defined(METALL_DISABLE_CXX17_FILESYSTEM_LIB) +inline bool fsync_recursive(const fs::path &path) { fs::path p(path); p = fs::canonical(p); while (true) { @@ -105,23 +81,6 @@ inline bool fsync_recursive(const std::string &path) { p = p.parent_path(); } return true; -#else - char *abs = ::realpath(path.c_str(), NULL); - if (!abs) return false; - char *ref = abs; - while (true) { - if (!fsync(std::string(abs))) { - ::free(ref); - return false; - } - if (::strcmp(abs, "/") == 0) { - break; - } - abs = ::dirname(abs); - } - ::free(ref); - return true; -#endif } inline bool extend_file_size_manually(const int fd, const off_t offset, @@ -130,7 +89,7 @@ inline bool extend_file_size_manually(const int fd, const off_t offset, for (off_t i = offset; i < file_size / 4096 + offset; ++i) { ::pwrite(fd, buffer, 4096, i * 4096); } - const size_t remained_size = file_size % 4096; + const std::size_t remained_size = file_size % 4096; if (remained_size > 0) ::pwrite(fd, buffer, remained_size, file_size - remained_size); @@ -141,7 +100,7 @@ inline bool extend_file_size_manually(const int fd, const off_t offset, return ret; } -inline bool extend_file_size(const int fd, const size_t file_size, +inline bool extend_file_size(const int fd, const std::size_t file_size, const bool fill_with_zero) { if (fill_with_zero) { #ifdef __APPLE__ @@ -157,8 +116,7 @@ inline bool extend_file_size(const int fd, const size_t file_size, } #endif } else { - // ----- extend the file if its size is smaller than that of mapped area - // ----- // + // extend the file if its size is smaller than that of mapped area struct stat stat_buf; if (::fstat(fd, &stat_buf) == -1) { logger::perror(logger::level::error, __FILE__, __LINE__, "fstat"); @@ -176,8 +134,8 @@ inline bool extend_file_size(const int fd, const size_t file_size, return ret; } -inline bool extend_file_size(const std::string &file_path, - const size_t file_size, +inline bool extend_file_size(const fs::path &file_path, + const std::size_t file_size, const bool fill_with_zero = false) { const int fd = ::open(file_path.c_str(), O_RDWR); if (fd == -1) { @@ -193,22 +151,23 @@ inline bool extend_file_size(const std::string &file_path, /// \brief Check if a file, any kinds of file including directory, exists /// \warning This implementation could return a wrong result due to metadata -/// cache on NFS. The following code could fail: if (mpi_rank == 1) -/// file_exist(path); // NFS creates metadata cache mpi_barrier(); if (mpi_rank -/// == 0) create_directory(path); mpi_barrier(); if (mpi_rank == 1) +/// cache on NFS. The following code could fail: +/// if (mpi_rank == 1) +/// file_exist(path); // NFS creates metadata cache +/// mpi_barrier(); +/// if (mpi_rank == 0) +/// create_directory(path); +/// mpi_barrier(); +/// if (mpi_rank == 1) /// assert(file_exist(path)); // Could fail due to the cached metadata. -inline bool file_exist(const std::string &file_name) { - std::string fixed_string(file_name); - while (fixed_string.back() == '/') { - fixed_string.pop_back(); - } - return (::access(fixed_string.c_str(), F_OK) == 0); +inline bool file_exist(const fs::path &file_name) { + return fs::exists(file_name); } /// \brief Check if a directory exists /// \warning This implementation could return a wrong result due to metadata /// cache on NFS. -inline bool directory_exist(const std::string &dir_path) { +inline bool directory_exist(const fs::path &dir_path) { struct stat stat_buf; if (::stat(dir_path.c_str(), &stat_buf) == -1) { return false; @@ -216,7 +175,7 @@ inline bool directory_exist(const std::string &dir_path) { return S_ISDIR(stat_buf.st_mode); } -inline bool create_file(const std::string &file_path) { +inline bool create_file(const fs::path &file_path) { const int fd = ::open(file_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); if (fd == -1) { @@ -231,13 +190,11 @@ inline bool create_file(const std::string &file_path) { return fsync_recursive(file_path); } -#if defined(__cpp_lib_filesystem) && \ - !defined(METALL_DISABLE_CXX17_FILESYSTEM_LIB) /// \brief Creates directories recursively. /// \return Returns true if the directory was created or already exists. /// Otherwise, returns false. -inline bool create_directory(const std::string &dir_path) { - std::string fixed_string = dir_path; +inline bool create_directory(const fs::path &dir_path) { + fs::path fixed_string = dir_path; // MEMO: GCC bug 87846 (fixed in v8.3) // "Calling std::filesystem::create_directories with a path with a trailing // separator (e.g. "./a/b/") does not create any directory." @@ -271,18 +228,8 @@ inline bool create_directory(const std::string &dir_path) { return success; } -#else -/// \brief Creates directories recursively. -/// \return Returns true if the directory was created or already exists, returns -/// true. Otherwise, returns false. -inline bool create_directory(const std::string &dir_path) { - std::string mkdir_command("mkdir -p " + dir_path); - const int status = std::system(mkdir_command.c_str()); - return (status != -1) && !!(WIFEXITED(status)); -} -#endif -inline ssize_t get_file_size(const std::string &file_path) { +inline ssize_t get_file_size(const fs::path &file_path) { std::ifstream ifs(file_path, std::ifstream::binary | std::ifstream::ate); ssize_t size = ifs.tellg(); if (size == -1) { @@ -297,31 +244,23 @@ inline ssize_t get_file_size(const std::string &file_path) { /// \brief /// Note that, according to GCC, /// the file system may use some blocks for internal record keeping -inline ssize_t get_actual_file_size(const std::string &file_path) { +inline ssize_t get_actual_file_size(const fs::path &file_path) { struct stat stat_buf; if (::stat(file_path.c_str(), &stat_buf) != 0) { - std::string s("stat (" + file_path + ")"); + std::string s("stat (" + file_path.string() + ")"); logger::perror(logger::level::error, __FILE__, __LINE__, s.c_str()); return -1; } - return stat_buf.st_blocks * 512LL; + return ssize_t(stat_buf.st_blocks) * ssize_t(stat_buf.st_blksize); } /// \brief Remove a file or directory /// \return Upon successful completion, returns true; otherwise, false is /// returned. If the file or directory does not exist, true is returned. -inline bool remove_file(const std::string &path) { -#if defined(__cpp_lib_filesystem) && \ - !defined(METALL_DISABLE_CXX17_FILESYSTEM_LIB) - std::filesystem::path p(path); +inline bool remove_file(const fs::path &path) { std::error_code ec; - [[maybe_unused]] const auto num_removed = std::filesystem::remove_all(p, ec); + std::filesystem::remove_all(path, ec); return !ec; -#else - std::string rm_command("rm -rf " + path); - const int status = std::system(rm_command.c_str()); - return (status != -1) && !!(WIFEXITED(status)); -#endif } inline bool free_file_space([[maybe_unused]] const int fd, @@ -343,12 +282,10 @@ inline bool free_file_space([[maybe_unused]] const int fd, #endif } -namespace file_copy_detail { +namespace fcpdtl { -#if defined(__cpp_lib_filesystem) && \ - !defined(METALL_DISABLE_CXX17_FILESYSTEM_LIB) -inline bool copy_file_dense(const std::string &source_path, - const std::string &destination_path) { +inline bool copy_file_dense(const fs::path &source_path, + const fs::path &destination_path) { bool success = true; try { if (!fs::copy_file(source_path, destination_path, @@ -371,77 +308,79 @@ inline bool copy_file_dense(const std::string &source_path, return success; } -#else - -inline bool copy_file_dense(const std::string &source_path, - const std::string &destination_path) { - { - const ssize_t source_file_size = get_file_size(source_path); - const ssize_t actual_source_file_size = get_actual_file_size(source_path); - if (source_file_size == -1 || actual_source_file_size == -1) { - return false; - } - - // If the source file is empty, just create the destination file and done. - if (source_file_size == 0 || actual_source_file_size == 0) { - create_file(destination_path); - return true; - } +inline bool copy_file_sparse_manually(const fs::path &source_path, + const fs::path &destination_path) { + const auto src_size = get_file_size(source_path); + if (src_size == -1) { + return false; + } + if (!extend_file_size(destination_path, src_size, false)) { + return false; } - { - std::ifstream source(source_path); - if (!source.is_open()) { + std::ifstream source; + std::ofstream dest; + const auto open_file = [](const auto &path, auto &file) { + file.open(path, std::ios::binary); + if (!file.is_open()) { std::stringstream ss; - ss << "Cannot open: " << source_path; + ss << "Failed to open file: " << path; logger::out(logger::level::error, __FILE__, __LINE__, ss.str().c_str()); return false; } + return true; + }; - std::ofstream destination(destination_path); - if (!destination.is_open()) { - std::stringstream ss; - ss << "Cannot open: " << destination_path; - logger::out(logger::level::error, __FILE__, __LINE__, ss.str().c_str()); - return false; - } + if (!open_file(source_path, source) || !open_file(destination_path, dest)) { + return false; + } - destination << source.rdbuf(); - if (!destination) { - std::stringstream ss; - ss << "Something happened in the ofstream: " << destination_path; - logger::out(logger::level::error, __FILE__, __LINE__, ss.str().c_str()); - return false; - } + const std::size_t block_size = 512; + char buffer[block_size]; - destination.close(); + const auto is_sparse = [](const char *const buffer, const std::size_t size) { + constexpr std::size_t chunk_size = sizeof(uint64_t); + const std::size_t num_chunks = size / chunk_size; + for (std::size_t i = 0; i < num_chunks; ++i) { + if (*reinterpret_cast(buffer + i * chunk_size) != + uint64_t(0)) { + return false; + } + } + // Check the remaining bytes + for (std::size_t i = num_chunks * chunk_size; i < size; ++i) { + if (buffer[i] != char(0)) { + return false; + } + } + return true; + }; - if (!metall::mtlldetail::fsync(destination_path)) { - return false; + while (source.read(buffer, block_size) || source.gcount() > 0) { + if (!is_sparse(buffer, source.gcount())) { + dest.write(buffer, source.gcount()); + } else { + dest.seekp(source.gcount(), std::ios_base::cur); } } - { - // Sanity check - const ssize_t s1 = get_file_size(source_path); - const ssize_t s2 = get_file_size(destination_path); - if (s1 < 0 || s1 != s2) { - std::stringstream ss; - ss << "Something wrong in file sizes: " << s1 << " " << s2; - logger::out(logger::level::error, __FILE__, __LINE__, ss.str().c_str()); - return false; - } + source.close(); + dest.close(); + if (!dest) { + std::stringstream ss; + ss << "Failed to write file: " << destination_path; + logger::out(logger::level::error, __FILE__, __LINE__, ss.str().c_str()); + return false; } + return true; } -#endif - #ifdef __linux__ -inline bool copy_file_sparse_linux(const std::string &source_path, - const std::string &destination_path) { - std::string command("cp --sparse=auto " + source_path + " " + - destination_path); +inline bool copy_file_sparse_linux(const fs::path &source_path, + const fs::path &destination_path) { + std::string command("cp --sparse=auto " + source_path.string() + " " + + destination_path.string()); const int status = std::system(command.c_str()); const bool success = (status != -1) && !!(WIFEXITED(status)); if (!success) { @@ -454,26 +393,25 @@ inline bool copy_file_sparse_linux(const std::string &source_path, } #endif -} // namespace file_copy_detail +} // namespace fcpdtl /// \brief Copy a file. /// \param source_path A source file path. /// \param destination_path A destination path. /// \param sparse_copy If true is specified, tries to perform sparse file copy. /// \return On success, returns true. On error, returns false. -inline bool copy_file(const std::string &source_path, - const std::string &destination_path, +inline bool copy_file(const fs::path &source_path, + const fs::path &destination_path, const bool sparse_copy = true) { if (sparse_copy) { #ifdef __linux__ - return file_copy_detail::copy_file_sparse_linux(source_path, - destination_path); + return fcpdtl::copy_file_sparse_linux(source_path, destination_path); #else logger::out(logger::level::info, __FILE__, __LINE__, "Sparse file copy is not available"); #endif } - return file_copy_detail::copy_file_dense(source_path, destination_path); + return fcpdtl::copy_file_dense(source_path, destination_path); } /// \brief Get the file names in a directory. @@ -483,11 +421,8 @@ inline bool copy_file(const std::string &source_path, /// \param file_list A buffer to put results. /// \return Returns true if there is no error (empty directory returns true as /// long as the operation does not fail). Returns false on error. -inline bool get_regular_file_names(const std::string &dir_path, - std::vector *file_list) { -#if defined(__cpp_lib_filesystem) && \ - !defined(METALL_DISABLE_CXX17_FILESYSTEM_LIB) - +inline bool get_regular_file_names(const fs::path &dir_path, + std::vector *file_list) { if (!directory_exist(dir_path)) { return false; } @@ -506,22 +441,6 @@ inline bool get_regular_file_names(const std::string &dir_path, } return true; - -#else - DIR *d = ::opendir(dir_path.c_str()); - if (!d) { - return false; - } - - for (dirent *dir; (dir = ::readdir(d)) != nullptr;) { - if (dir->d_type == DT_REG) { - file_list->push_back(dir->d_name); - } - } - ::closedir(d); - - return true; -#endif } /// \brief Copy files in a directory. @@ -534,11 +453,10 @@ inline bool get_regular_file_names(const std::string &dir_path, /// \param copy_func The actual copy function. /// \return On success, returns true. On error, returns false. inline bool copy_files_in_directory_in_parallel_helper( - const std::string &source_dir_path, const std::string &destination_dir_path, + const fs::path &source_dir_path, const fs::path &destination_dir_path, const int max_num_threads, - const std::function - ©_func) { - std::vector src_file_names; + const std::function ©_func) { + std::vector src_file_names; if (!get_regular_file_names(source_dir_path, &src_file_names)) { std::stringstream ss; ss << "Failed to get file list in " << source_dir_path; @@ -554,10 +472,9 @@ inline bool copy_files_in_directory_in_parallel_helper( while (true) { const auto file_no = file_no_cnt.fetch_add(1); if (file_no >= src_file_names.size()) break; - const std::string &src_file_path = - source_dir_path + "/" + src_file_names[file_no]; - const std::string &dst_file_path = - destination_dir_path + "/" + src_file_names[file_no]; + const fs::path &src_file_path = source_dir_path / src_file_names[file_no]; + const fs::path &dst_file_path = + destination_dir_path / src_file_names[file_no]; num_successes.fetch_add(copy_func(src_file_path, dst_file_path) ? 1 : 0); } }; @@ -587,11 +504,11 @@ inline bool copy_files_in_directory_in_parallel_helper( /// \param sparse_copy Performs sparse file copy. /// \return On success, returns true. On error, returns false. inline bool copy_files_in_directory_in_parallel( - const std::string &source_dir_path, const std::string &destination_dir_path, + const fs::path &source_dir_path, const fs::path &destination_dir_path, const int max_num_threads, const bool sparse_copy = true) { return copy_files_in_directory_in_parallel_helper( source_dir_path, destination_dir_path, max_num_threads, - [&sparse_copy](const std::string &src, const std::string &dst) -> bool { + [&sparse_copy](const fs::path &src, const fs::path &dst) -> bool { return copy_file(src, dst, sparse_copy); }); } diff --git a/include/metall/detail/file_clone.hpp b/include/metall/detail/file_clone.hpp index f4cdf8df..81d3d071 100644 --- a/include/metall/detail/file_clone.hpp +++ b/include/metall/detail/file_clone.hpp @@ -23,21 +23,26 @@ namespace metall::mtlldetail { +namespace { +namespace fs = std::filesystem; +} + namespace file_clone_detail { #ifdef __linux__ -inline bool clone_file_linux(const std::string &source_path, - const std::string &destination_path) { - std::string command("cp --reflink=auto -R " + source_path + " " + - destination_path); +inline bool clone_file_linux(const fs::path &source_path, + const fs::path &destination_path) { + std::string command("cp --reflink=auto -R " + source_path.string() + " " + + destination_path.string()); const int status = std::system(command.c_str()); return (status != -1) && !!(WIFEXITED(status)); } #endif #ifdef __APPLE__ -inline bool clone_file_macos(const std::string &source_path, - const std::string &destination_path) { - std::string command("cp -cR " + source_path + " " + destination_path); +inline bool clone_file_macos(const fs::path &source_path, + const fs::path &destination_path) { + std::string command("cp -cR " + source_path.string() + " " + + destination_path.string()); const int status = std::system(command.c_str()); return (status != -1) && !!(WIFEXITED(status)); } @@ -48,21 +53,21 @@ inline bool clone_file_macos(const std::string &source_path, /// normally. \param source_path A path to the file to be cloned. \param /// destination_path A path to copy to. \return On success, returns true. On /// error, returns false. -inline bool clone_file(const std::string &source_path, - const std::string &destination_path) { +inline bool clone_file(const fs::path &source_path, + const fs::path &destination_path) { bool ret = false; #if defined(__linux__) ret = file_clone_detail::clone_file_linux(source_path, destination_path); if (!ret) { - std::string s("On Linux, Failed to clone " + source_path + " to " + - destination_path); + std::string s("On Linux, Failed to clone " + source_path.string() + " to " + + destination_path.string()); logger::out(logger::level::error, __FILE__, __LINE__, s.c_str()); } #elif defined(__APPLE__) ret = file_clone_detail::clone_file_macos(source_path, destination_path); if (!ret) { - std::string s("On MacOS, Failed to clone " + source_path + " to " + - destination_path); + std::string s("On MacOS, Failed to clone " + source_path.string() + " to " + + destination_path.string()); logger::out(logger::level::error, __FILE__, __LINE__, s.c_str()); } #else @@ -73,7 +78,8 @@ inline bool clone_file(const std::string &source_path, "Use normal copy instead of clone"); ret = copy_file(source_path, destination_path); // Copy normally if (!ret) { - std::string s("Failed to copy " + source_path + " to " + destination_path); + std::string s("Failed to copy " + source_path.string() + " to " + + destination_path.string()); logger::out(logger::level::error, __FILE__, __LINE__, s.c_str()); } #endif @@ -93,7 +99,7 @@ inline bool clone_file(const std::string &source_path, /// If <= 0 is given, it is automatically determined. /// \return On success, returns true. On error, returns false. inline bool clone_files_in_directory_in_parallel( - const std::string &source_dir_path, const std::string &destination_dir_path, + const fs::path &source_dir_path, const fs::path &destination_dir_path, const int max_num_threads) { return copy_files_in_directory_in_parallel_helper( source_dir_path, destination_dir_path, max_num_threads, clone_file); diff --git a/include/metall/detail/mmap.hpp b/include/metall/detail/mmap.hpp index 5e244c67..439d7539 100644 --- a/include/metall/detail/mmap.hpp +++ b/include/metall/detail/mmap.hpp @@ -98,7 +98,7 @@ inline void *map_anonymous_write_mode(void *const addr, const size_t length, /// \return A pair of the file descriptor of the file and the starting address /// for the map inline std::pair map_file_read_mode( - const std::string &file_name, void *const addr, const size_t length, + const fs::path &file_name, void *const addr, const size_t length, const off_t offset, const int additional_flags = 0) { // ----- Open the file ----- // const int fd = ::open(file_name.c_str(), O_RDONLY); @@ -147,7 +147,7 @@ inline void *map_file_write_mode(const int fd, void *const addr, /// \return A pair of the file descriptor of the file and the starting address /// for the map inline std::pair map_file_write_mode( - const std::string &file_name, void *const addr, const size_t length, + const fs::path &file_name, void *const addr, const size_t length, const off_t offset, const int additional_flags = 0) { // ----- Open the file ----- // const int fd = ::open(file_name.c_str(), O_RDWR); @@ -197,7 +197,7 @@ inline void *map_file_write_private_mode(const int fd, void *const addr, /// \return A pair of the file descriptor of the file and the starting address /// for the map inline std::pair map_file_write_private_mode( - const std::string &file_name, void *const addr, const size_t length, + const fs::path &file_name, void *const addr, const size_t length, const off_t offset, const int additional_flags = 0) { // ----- Open the file ----- // const int fd = ::open(file_name.c_str(), O_RDWR); diff --git a/include/metall/detail/ptree.hpp b/include/metall/detail/ptree.hpp index e547fb7c..13ef5e74 100644 --- a/include/metall/detail/ptree.hpp +++ b/include/metall/detail/ptree.hpp @@ -6,6 +6,9 @@ #ifndef METALL_DETAIL_UTILITY_PTREE_HPP #define METALL_DETAIL_UTILITY_PTREE_HPP +#include +#include + #include #include @@ -14,6 +17,7 @@ namespace metall::mtlldetail::ptree { namespace { +namespace fs = std::filesystem; namespace bptree = boost::property_tree; } @@ -117,7 +121,7 @@ inline bool push_back(const node_type &child, node_type *parent) { return true; } -inline bool read_json(const std::string &file_name, node_type *root) { +inline bool read_json(const fs::path &file_name, node_type *root) { try { bptree::read_json(file_name, *root); } catch (const bptree::json_parser_error &e) { @@ -127,7 +131,7 @@ inline bool read_json(const std::string &file_name, node_type *root) { return true; } -inline bool write_json(const node_type &root, const std::string &file_name) { +inline bool write_json(const node_type &root, const fs::path &file_name) { try { bptree::write_json(file_name, root); } catch (const bptree::json_parser_error &e) { diff --git a/include/metall/kernel/attributed_object_directory.hpp b/include/metall/kernel/attributed_object_directory.hpp index 3d85d5b4..83075b75 100644 --- a/include/metall/kernel/attributed_object_directory.hpp +++ b/include/metall/kernel/attributed_object_directory.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -28,6 +29,7 @@ namespace metall { namespace kernel { namespace { +namespace fs = std::filesystem; namespace mdtl = metall::mtlldetail; namespace json = metall::mtlldetail::ptree; } // namespace @@ -429,7 +431,7 @@ class attributed_object_directory { /// \brief /// \param path - bool serialize(const char *const path) const noexcept { + bool serialize(const fs::path &path) const noexcept { try { return priv_serialize_throw(path); } catch (...) { @@ -440,7 +442,7 @@ class attributed_object_directory { /// \brief /// \param path - bool deserialize(const char *const path) noexcept { + bool deserialize(const fs::path &path) noexcept { try { return priv_deserialize_throw(path); } catch (...) { @@ -507,7 +509,7 @@ class attributed_object_directory { return true; } - bool priv_serialize_throw(const char *const path) const { + bool priv_serialize_throw(const fs::path &path) const { if (!good()) { return false; } @@ -546,7 +548,7 @@ class attributed_object_directory { return true; } - bool priv_deserialize_throw(const char *const path) { + bool priv_deserialize_throw(const fs::path &path) { if (!good()) { return false; } diff --git a/include/metall/kernel/bin_directory.hpp b/include/metall/kernel/bin_directory.hpp index e3b8bb29..f96f2d58 100644 --- a/include/metall/kernel/bin_directory.hpp +++ b/include/metall/kernel/bin_directory.hpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,7 @@ namespace metall { namespace kernel { namespace { +namespace fs = std::filesystem; namespace mdtl = metall::mtlldetail; } @@ -195,7 +197,7 @@ class bin_directory { /// \brief /// \param path - bool serialize(const char *path) const { + bool serialize(const fs::path &path) const { std::ofstream ofs(path); if (!ofs.is_open()) { std::stringstream ss; @@ -224,7 +226,7 @@ class bin_directory { /// \brief /// \param path - bool deserialize(const char *path) { + bool deserialize(const fs::path &path) { std::ifstream ifs(path); if (!ifs.is_open()) { std::stringstream ss; diff --git a/include/metall/kernel/chunk_directory.hpp b/include/metall/kernel/chunk_directory.hpp index d5174232..0cc70e8e 100644 --- a/include/metall/kernel/chunk_directory.hpp +++ b/include/metall/kernel/chunk_directory.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -22,8 +23,9 @@ namespace metall { namespace kernel { namespace { +namespace fs = std::filesystem; namespace mdtl = metall::mtlldetail; -} +} // namespace /// \brief Chunk directory class. /// Chunk directory is a table that stores information about chunks. @@ -294,7 +296,7 @@ class chunk_directory { /// \brief /// \param path - bool serialize(const char *path) const { + bool serialize(const fs::path &path) const { std::ofstream ofs(path); if (!ofs.is_open()) { std::stringstream ss; @@ -358,7 +360,7 @@ class chunk_directory { /// \brief /// \param path /// \return - bool deserialize(const char *path) { + bool deserialize(const fs::path &path) { std::ifstream ifs(path); if (!ifs.is_open()) { std::stringstream ss; diff --git a/include/metall/kernel/segment_allocator.hpp b/include/metall/kernel/segment_allocator.hpp index 7e601cea..0e53f1c8 100644 --- a/include/metall/kernel/segment_allocator.hpp +++ b/include/metall/kernel/segment_allocator.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -40,8 +41,9 @@ namespace metall { namespace kernel { namespace { +namespace fs = std::filesystem; namespace mdtl = metall::mtlldetail; -} +} // namespace template +#include + #include int main(int argc, char *argv[]) { @@ -12,7 +14,7 @@ int main(int argc, char *argv[]) { std::abort(); } - const std::string datastore_path = argv[1]; + const std::filesystem::path datastore_path = argv[1]; metall::utility::ls_named_object(datastore_path); std::cout << std::endl; diff --git a/src/mpi_datastore_ls.cpp b/src/mpi_datastore_ls.cpp index 57b9dd98..9ff1a43a 100644 --- a/src/mpi_datastore_ls.cpp +++ b/src/mpi_datastore_ls.cpp @@ -5,6 +5,8 @@ #include #include +#include + #include #include @@ -14,14 +16,14 @@ int main(int argc, char *argv[]) { std::abort(); } - const std::string datastore_path = argv[1]; + const std::filesystem::path datastore_path = argv[1]; const int mpi_rank = (argc < 3) ? 0 : std::stoi(argv[2]); const auto local_datastore_path = metall::utility::mpi_datastore::make_local_dir_path(datastore_path, mpi_rank); - if (!metall::manager::consistent(local_datastore_path.c_str())) { + if (!metall::manager::consistent(local_datastore_path)) { std::cerr << "Inconsistent datastore or invalid datastore path" << std::endl; std::abort(); diff --git a/test/container/concurrent_map_test.cpp b/test/container/concurrent_map_test.cpp index 16fc3943..a14d61a3 100644 --- a/test/container/concurrent_map_test.cpp +++ b/test/container/concurrent_map_test.cpp @@ -5,6 +5,8 @@ #include "gtest/gtest.h" +#include + #include #include #include @@ -130,7 +132,7 @@ TEST(ConcurrentMapTest, Persistence) { metall::container::concurrent_map, std::hash, allocator_type, 2>; - const std::string file_path(test_utility::make_test_path()); + const std::filesystem::path file_path(test_utility::make_test_path()); test_utility::create_test_dir(); metall::mtlldetail::remove_file(file_path); diff --git a/test/container/fallback_allocator_adaptor_test.cpp b/test/container/fallback_allocator_adaptor_test.cpp index 34d0e6b3..4a1443a9 100644 --- a/test/container/fallback_allocator_adaptor_test.cpp +++ b/test/container/fallback_allocator_adaptor_test.cpp @@ -199,7 +199,7 @@ TEST(FallbackAllocatorAdaptorTest, PersistentConstructFind) { boost::interprocess::vector>; { - metall::manager manager(metall::create_only, dir_path().c_str(), + metall::manager manager(metall::create_only, dir_path(), 1UL << 27UL); int *a = manager.construct("int")(10); @@ -212,7 +212,7 @@ TEST(FallbackAllocatorAdaptorTest, PersistentConstructFind) { } { - metall::manager manager(metall::open_only, dir_path().c_str()); + metall::manager manager(metall::open_only, dir_path()); const auto ret1 = manager.find("int"); ASSERT_NE(ret1.first, nullptr); diff --git a/test/container/json/json_value.cpp b/test/container/json/json_value.cpp index d2d6c279..f04ab36b 100644 --- a/test/container/json/json_value.cpp +++ b/test/container/json/json_value.cpp @@ -365,10 +365,10 @@ TEST(JSONValueTest, CopyDifferentMetallAllocator) { { metall::manager manager_copy( metall::create_only, - (test_utility::make_test_path() + "_copy").c_str()); + (test_utility::make_test_path().string() + "_copy")); metall::manager manager_src( metall::create_only, - (test_utility::make_test_path() + "_src").c_str()); + (test_utility::make_test_path().string() + "_src")); auto *jv_copy = manager_copy.construct("jv")(manager_copy.get_allocator()); @@ -380,7 +380,7 @@ TEST(JSONValueTest, CopyDifferentMetallAllocator) { { metall::manager manager_copy( metall::open_read_only, - (test_utility::make_test_path() + "_copy").c_str()); + (test_utility::make_test_path().string() + "_copy")); const auto *const jv_copy = manager_copy.find("jv").first; check_json_string(*jv_copy); } @@ -391,10 +391,10 @@ TEST(JSONValueTest, CopyDifferentMetallAllocator) { { metall::manager manager_src( metall::create_only, - (test_utility::make_test_path() + "_src").c_str()); + (test_utility::make_test_path().string() + "_src")); metall::manager manager_copy( metall::create_only, - (test_utility::make_test_path() + "_copy").c_str()); + (test_utility::make_test_path().string() + "_copy")); auto json_src = mj::parse(json_string, manager_src.get_allocator()); // Construct a new one from another instance that was allocated by another // allocator @@ -406,7 +406,7 @@ TEST(JSONValueTest, CopyDifferentMetallAllocator) { { metall::manager manager_copy( metall::open_read_only, - (test_utility::make_test_path() + "_copy").c_str()); + (test_utility::make_test_path().string() + "_copy")); const auto *const jv_copy = manager_copy.find("jv").first; check_json_string(*jv_copy); } @@ -420,10 +420,10 @@ TEST(JSONValueTest, MoveDifferentMetallAllocator) { { metall::manager manager_move( metall::create_only, - (test_utility::make_test_path() + "_move").c_str()); + (test_utility::make_test_path().string() + "_move")); metall::manager manager_src( metall::create_only, - (test_utility::make_test_path() + "_src").c_str()); + (test_utility::make_test_path().string() + "_src")); auto *jv_move = manager_move.construct("jv")(manager_move.get_allocator()); @@ -434,7 +434,7 @@ TEST(JSONValueTest, MoveDifferentMetallAllocator) { { metall::manager manager_move( metall::open_read_only, - (test_utility::make_test_path() + "_move").c_str()); + (test_utility::make_test_path().string() + "_move")); const auto *const jv_move = manager_move.find("jv").first; check_json_string(*jv_move); } @@ -445,10 +445,10 @@ TEST(JSONValueTest, MoveDifferentMetallAllocator) { { metall::manager manager_src( metall::create_only, - (test_utility::make_test_path() + "_src").c_str()); + (test_utility::make_test_path().string() + "_src")); metall::manager manager_move( metall::create_only, - (test_utility::make_test_path() + "_move").c_str()); + (test_utility::make_test_path().string() + "_move")); // Construct a new one from another instance that was allocated by another // allocator auto *jv_move = manager_move.construct("jv")( @@ -460,7 +460,7 @@ TEST(JSONValueTest, MoveDifferentMetallAllocator) { { metall::manager manager_move( metall::open_read_only, - (test_utility::make_test_path() + "_move").c_str()); + (test_utility::make_test_path().string() + "_move")); const auto *const jv_move = manager_move.find("jv").first; check_json_string(*jv_move); } diff --git a/test/container/stl_allocator_test.cpp b/test/container/stl_allocator_test.cpp index 1a3759fd..ebce177a 100644 --- a/test/container/stl_allocator_test.cpp +++ b/test/container/stl_allocator_test.cpp @@ -4,19 +4,25 @@ // SPDX-License-Identifier: (Apache-2.0 OR MIT) #include "gtest/gtest.h" + #include #include +#include + #include #include #include #include #include "../test_utility.hpp" +namespace { +namespace fs = std::filesystem; + template using alloc_type = metall::manager::allocator_type; -const std::string &dir_path() { - const static std::string path(test_utility::make_test_path()); +const fs::path &dir_path() { + const static fs::path path(test_utility::make_test_path()); return path; } @@ -100,8 +106,7 @@ TEST(StlAllocatorTest, Types) { }; using alloc_t = alloc_type; - metall::manager manager(metall::create_only, dir_path().c_str(), - 1UL << 24UL); + metall::manager manager(metall::create_only, dir_path(), 1UL << 24UL); alloc_t alloc = manager.get_allocator(); { @@ -130,7 +135,7 @@ TEST(StlAllocatorTest, Types) { } TEST(StlAllocatorTest, Exception) { - metall::manager manager(metall::create_only, dir_path().c_str(), 1UL << 24UL); + metall::manager manager(metall::create_only, dir_path(), 1UL << 24UL); alloc_type allocator = manager.get_allocator(); @@ -140,16 +145,16 @@ TEST(StlAllocatorTest, Exception) { ASSERT_NO_THROW({ allocator.deallocate(allocator.allocate(1), 1); }); - ASSERT_THROW({ allocator.allocate(allocator.max_size() + 1); }, - std::bad_array_new_length); + ASSERT_THROW( + { allocator.allocate(allocator.max_size() + 1); }, + std::bad_array_new_length); metall::logger::set_log_level(metall::logger::level::error); } TEST(StlAllocatorTest, Container) { { - metall::manager manager(metall::create_only, dir_path().c_str(), - 1UL << 27UL); + metall::manager manager(metall::create_only, dir_path(), 1UL << 27UL); using element_type = std::pair; boost::interprocess::vector> vector( @@ -176,8 +181,7 @@ TEST(StlAllocatorTest, NestedContainer) { alloc_type>>>; { - metall::manager manager(metall::create_only, dir_path().c_str(), - 1UL << 27UL); + metall::manager manager(metall::create_only, dir_path(), 1UL << 27UL); map_type map(manager.get_allocator<>()); for (uint64_t i = 0; i < 1024; ++i) { @@ -196,8 +200,7 @@ TEST(StlAllocatorTest, PersistentConstructFind) { boost::interprocess::vector>; { - metall::manager manager(metall::create_only, dir_path().c_str(), - 1UL << 27UL); + metall::manager manager(metall::create_only, dir_path(), 1UL << 27UL); int *a = manager.construct("int")(10); ASSERT_EQ(*a, 10); @@ -209,7 +212,7 @@ TEST(StlAllocatorTest, PersistentConstructFind) { } { - metall::manager manager(metall::open_only, dir_path().c_str()); + metall::manager manager(metall::open_only, dir_path()); const auto ret1 = manager.find("int"); ASSERT_NE(ret1.first, nullptr); @@ -226,7 +229,7 @@ TEST(StlAllocatorTest, PersistentConstructFind) { } { - metall::manager manager(metall::open_only, dir_path().c_str()); + metall::manager manager(metall::open_only, dir_path()); ASSERT_TRUE(manager.destroy("int")); ASSERT_FALSE(manager.destroy("int")); @@ -241,8 +244,7 @@ TEST(StlAllocatorTest, PersistentConstructOrFind) { boost::interprocess::vector>; { - metall::manager manager(metall::create_only, dir_path().c_str(), - 1UL << 27UL); + metall::manager manager(metall::create_only, dir_path(), 1UL << 27UL); int *a = manager.find_or_construct("int")(10); ASSERT_EQ(*a, 10); @@ -253,7 +255,7 @@ TEST(StlAllocatorTest, PersistentConstructOrFind) { } { - metall::manager manager(metall::open_only, dir_path().c_str()); + metall::manager manager(metall::open_only, dir_path()); int *a = manager.find_or_construct("int")(20); ASSERT_EQ(*a, 10); @@ -265,7 +267,7 @@ TEST(StlAllocatorTest, PersistentConstructOrFind) { } { - metall::manager manager(metall::open_only, dir_path().c_str()); + metall::manager manager(metall::open_only, dir_path()); ASSERT_TRUE(manager.destroy("int")); ASSERT_FALSE(manager.destroy("int")); @@ -287,8 +289,7 @@ TEST(StlAllocatorTest, PersistentNestedContainer) { alloc_type>>>; { - metall::manager manager(metall::create_only, dir_path().c_str(), - 1UL << 27UL); + metall::manager manager(metall::create_only, dir_path(), 1UL << 27UL); map_type *map = manager.construct("map")(manager.get_allocator<>()); (*map)[0].emplace_back(1); @@ -296,7 +297,7 @@ TEST(StlAllocatorTest, PersistentNestedContainer) { } { - metall::manager manager(metall::open_only, dir_path().c_str()); + metall::manager manager(metall::open_only, dir_path()); map_type *map; std::size_t n; std::tie(map, n) = manager.find("map"); @@ -307,7 +308,7 @@ TEST(StlAllocatorTest, PersistentNestedContainer) { } { - metall::manager manager(metall::open_read_only, dir_path().c_str()); + metall::manager manager(metall::open_read_only, dir_path()); map_type *map; std::size_t n; std::tie(map, n) = manager.find("map"); @@ -316,4 +317,5 @@ TEST(StlAllocatorTest, PersistentNestedContainer) { ASSERT_EQ(map->at(0)[1], 2); ASSERT_EQ(map->at(1)[0], 3); } -} \ No newline at end of file +} +} // namespace \ No newline at end of file diff --git a/test/kernel/attributed_object_directory_test.cpp b/test/kernel/attributed_object_directory_test.cpp index c5b59750..3ee0cb87 100644 --- a/test/kernel/attributed_object_directory_test.cpp +++ b/test/kernel/attributed_object_directory_test.cpp @@ -176,7 +176,7 @@ TEST(AttributedObjectDirectoryTest, Serialize) { test_utility::create_test_dir(); const auto file(test_utility::make_test_path()); - ASSERT_TRUE(obj.serialize(file.c_str())); + ASSERT_TRUE(obj.serialize(file)); } TEST(AttributedObjectDirectoryTest, Deserialize) { @@ -187,12 +187,12 @@ TEST(AttributedObjectDirectoryTest, Deserialize) { directory_type obj; obj.insert("item1", 1, 2, 5); obj.insert("item2", 3, 4, 6, "description2"); - obj.serialize(file.c_str()); + obj.serialize(file); } { directory_type obj; - ASSERT_TRUE(obj.deserialize(file.c_str())); + ASSERT_TRUE(obj.deserialize(file)); // Get values correctly const auto itr1 = obj.find("item1"); diff --git a/test/kernel/bin_directory_test.cpp b/test/kernel/bin_directory_test.cpp index b1c38a9f..624e5d51 100644 --- a/test/kernel/bin_directory_test.cpp +++ b/test/kernel/bin_directory_test.cpp @@ -98,7 +98,7 @@ TEST(BinDirectoryTest, Serialize) { test_utility::create_test_dir(); const auto file = test_utility::make_test_path(); - ASSERT_TRUE(obj.serialize(file.c_str())); + ASSERT_TRUE(obj.serialize(file)); } TEST(BinDirectoryTest, Deserialize) { @@ -114,13 +114,13 @@ TEST(BinDirectoryTest, Deserialize) { obj.insert(num_small_bins - 1, 3); obj.insert(num_small_bins - 1, 4); - obj.serialize(file.c_str()); + obj.serialize(file); } { std::allocator allocator; directory_type obj(allocator); - ASSERT_TRUE(obj.deserialize(file.c_str())); + ASSERT_TRUE(obj.deserialize(file)); #ifdef METALL_USE_SORTED_BIN ASSERT_EQ(obj.front(0), 1); diff --git a/test/kernel/chunk_directory_test.cpp b/test/kernel/chunk_directory_test.cpp index 5e26bc39..8d8fdc64 100644 --- a/test/kernel/chunk_directory_test.cpp +++ b/test/kernel/chunk_directory_test.cpp @@ -124,7 +124,7 @@ TEST(ChunkDirectoryTest, Serialize) { test_utility::create_test_dir(); const auto file(test_utility::make_test_path()); - ASSERT_TRUE(directory.serialize(file.c_str())); + ASSERT_TRUE(directory.serialize(file)); } TEST(ChunkDirectoryTest, Deserialize) { @@ -146,13 +146,13 @@ TEST(ChunkDirectoryTest, Deserialize) { directory.insert(bin_no_mngr::num_small_bins()); // 1 chunk directory.insert(bin_no_mngr::num_small_bins() + 1); // 2 chunks - directory.serialize(file.c_str()); + directory.serialize(file); } { chunk_directory_type directory(bin_no_mngr::num_small_bins() + 4); - ASSERT_TRUE(directory.deserialize(file.c_str())); + ASSERT_TRUE(directory.deserialize(file)); for (uint64_t i = 0; i < bin_no_mngr::num_small_bins(); ++i) { const auto bin_no = static_cast(i); const auto chunk_no = static_cast(i); diff --git a/test/kernel/manager_multithread_test.cpp b/test/kernel/manager_multithread_test.cpp index 46138ed4..6e2a4ecd 100644 --- a/test/kernel/manager_multithread_test.cpp +++ b/test/kernel/manager_multithread_test.cpp @@ -97,7 +97,7 @@ template void run_alloc_dealloc_separated_test(const list_type &allocation_size_list) { // Allocate manager const auto dir(test_utility::make_test_path()); - manager_type manager(metall::create_only, dir.c_str()); + manager_type manager(metall::create_only, dir); // Main loop std::pair previous_allocation_rage(nullptr, nullptr); @@ -132,7 +132,7 @@ void run_alloc_dealloc_mixed_and_write_value_test( const list_type &allocation_size_list) { // Allocate manager const auto dir(test_utility::make_test_path()); - manager_type manager(metall::create_only, dir.c_str()); + manager_type manager(metall::create_only, dir); // Main loop std::vector> previous_addr_and_size_array( @@ -293,7 +293,7 @@ TEST(ManagerMultithreadsTest, ConstructAndFind) { constexpr std::size_t num_allocates = 1024; const auto dir(test_utility::make_test_path()); - manager_type manager(metall::create_only, dir.c_str()); + manager_type manager(metall::create_only, dir); std::vector keys; for (std::size_t i = 0; i < num_allocates; ++i) { diff --git a/test/kernel/manager_test.cpp b/test/kernel/manager_test.cpp index 491c8bb3..7c728338 100644 --- a/test/kernel/manager_test.cpp +++ b/test/kernel/manager_test.cpp @@ -5,6 +5,7 @@ #include "gtest/gtest.h" +#include #include #include @@ -12,6 +13,8 @@ #include "../test_utility.hpp" namespace { +namespace fs = std::filesystem; + using namespace metall::mtlldetail; using manager_type = metall::manager; @@ -23,23 +26,21 @@ using object_size_mngr = metall::kernel::object_size_manager; constexpr std::size_t k_min_object_size = object_size_mngr::at(0); -const std::string &dir_path() { - const static std::string path(test_utility::make_test_path()); +const fs::path &dir_path() { + const static fs::path path(test_utility::make_test_path()); return path; } TEST(ManagerTest, CreateAndOpenModes) { { - manager_type::remove(dir_path().c_str()); + manager_type::remove(dir_path()); { - manager_type manager(metall::create_only, dir_path().c_str(), - 1UL << 30UL); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_NE(manager.construct("int")(10), nullptr); ASSERT_TRUE(manager.destroy("int")); } { - manager_type manager(metall::create_only, dir_path().c_str(), - 1UL << 30UL); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); auto ret = manager.find("int"); ASSERT_EQ(ret.first, nullptr); ASSERT_FALSE(manager.destroy("int")); @@ -47,14 +48,13 @@ TEST(ManagerTest, CreateAndOpenModes) { } { - manager_type::remove(dir_path().c_str()); + manager_type::remove(dir_path()); { - manager_type manager(metall::create_only, dir_path().c_str(), - 1UL << 30UL); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_NE(manager.construct("int")(10), nullptr); } { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); auto ret = manager.find("int"); ASSERT_NE(ret.first, nullptr); ASSERT_EQ(*(static_cast(ret.first)), 10); @@ -63,20 +63,19 @@ TEST(ManagerTest, CreateAndOpenModes) { } { - manager_type::remove(dir_path().c_str()); + manager_type::remove(dir_path()); { - manager_type manager(metall::create_only, dir_path().c_str(), - 1UL << 30UL); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_NE(manager.construct("int")(10), nullptr); } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); auto ret = manager.find("int"); ASSERT_NE(ret.first, nullptr); ASSERT_EQ(*(static_cast(ret.first)), 10); } { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); auto ret = manager.find("int"); ASSERT_NE(ret.first, nullptr); ASSERT_EQ(*(static_cast(ret.first)), 10); @@ -88,14 +87,13 @@ TEST(ManagerTest, CreateAndOpenModes) { TEST(ManagerTest, ConstructArray) { { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), - 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_NE(manager.construct("int")[2](10), nullptr); } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); auto ret = manager.find("int"); ASSERT_NE(ret.first, nullptr); ASSERT_EQ(ret.second, 2); @@ -105,7 +103,7 @@ TEST(ManagerTest, ConstructArray) { } { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); ASSERT_TRUE(manager.destroy("int")); } } @@ -114,20 +112,19 @@ TEST(ManagerTest, ConstructArray) { TEST(ManagerTest, findOrConstruct) { { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), - 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_NE(manager.find_or_construct("int")(10), nullptr); } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); int *a = manager.find_or_construct("int")(20); ASSERT_EQ(*a, 10); } { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); ASSERT_TRUE(manager.destroy("int")); } } @@ -136,21 +133,20 @@ TEST(ManagerTest, findOrConstruct) { TEST(ManagerTest, findOrConstructArray) { { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), - 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_NE(manager.find_or_construct("int")[2](10), nullptr); } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); int *a = manager.find_or_construct("int")[2](20); ASSERT_EQ(a[0], 10); ASSERT_EQ(a[1], 10); } { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); ASSERT_TRUE(manager.destroy("int")); } } @@ -160,16 +156,15 @@ TEST(ManagerTest, ConstructContainers) { { using vec_t = std::vector>; { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), - 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_NE(manager.construct("vecs")[2]( 2, 10, manager.get_allocator()), nullptr); } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); auto ret = manager.find("vecs"); ASSERT_NE(ret.first, nullptr); ASSERT_EQ(ret.second, 2); @@ -183,7 +178,7 @@ TEST(ManagerTest, ConstructContainers) { } { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); ASSERT_TRUE(manager.destroy("vecs")); ASSERT_TRUE(manager.all_memory_deallocated()); } @@ -193,15 +188,14 @@ TEST(ManagerTest, ConstructContainers) { TEST(ManagerTest, ConstructWithIterator) { { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), - 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); int values[2] = {10, 20}; ASSERT_NE(manager.construct_it("int")[2](&values[0]), nullptr); } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); auto ret = manager.find("int"); ASSERT_NE(ret.first, nullptr); ASSERT_EQ(ret.second, 2); @@ -211,7 +205,7 @@ TEST(ManagerTest, ConstructWithIterator) { } { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); ASSERT_TRUE(manager.destroy("int")); } } @@ -226,9 +220,8 @@ TEST(ManagerTest, ConstructObjectsWithIterator) { { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), - 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); int values1[2] = {10, 20}; float values2[2] = {0.1, 0.2}; ASSERT_NE(manager.construct_it("data")[2](&values1[0], &values2[0]), @@ -236,7 +229,7 @@ TEST(ManagerTest, ConstructObjectsWithIterator) { } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); auto ret = manager.find("data"); ASSERT_NE(ret.first, nullptr); ASSERT_EQ(ret.second, 2); @@ -248,7 +241,7 @@ TEST(ManagerTest, ConstructObjectsWithIterator) { } { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); ASSERT_TRUE(manager.destroy("data")); } } @@ -257,16 +250,15 @@ TEST(ManagerTest, ConstructObjectsWithIterator) { TEST(ManagerTest, FindOrConstructWithIterator) { { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), - 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); int values[2] = {10, 20}; ASSERT_NE(manager.find_or_construct_it("int")[2](&values[0]), nullptr); } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); int values[2] = {30, 40}; int *a = manager.find_or_construct_it("int")[2](&values[0]); ASSERT_NE(a, nullptr); @@ -275,7 +267,7 @@ TEST(ManagerTest, FindOrConstructWithIterator) { } { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); ASSERT_TRUE(manager.destroy("int")); } } @@ -283,8 +275,8 @@ TEST(ManagerTest, FindOrConstructWithIterator) { TEST(ManagerTest, Destroy) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_FALSE(manager.destroy("named_obj")); ASSERT_FALSE(manager.destroy(metall::unique_instance)); @@ -307,8 +299,8 @@ TEST(ManagerTest, Destroy) { } { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); manager.construct("named_obj")(); manager.construct(metall::unique_instance)(); @@ -317,7 +309,7 @@ TEST(ManagerTest, Destroy) { // Destroy after restoring { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); ASSERT_TRUE(manager.destroy("named_obj")); ASSERT_TRUE(manager.destroy(metall::unique_instance)); @@ -329,8 +321,8 @@ TEST(ManagerTest, Destroy) { TEST(ManagerTest, DestroyPtr) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); int *named_obj = manager.construct("named_obj")(); int *unique_obj = manager.construct(metall::unique_instance)(); @@ -351,8 +343,8 @@ TEST(ManagerTest, DestroyPtr) { } { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); manager.construct("named_obj")(); manager.construct(metall::unique_instance)(); @@ -364,7 +356,7 @@ TEST(ManagerTest, DestroyPtr) { // Destroy after restoring { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); ASSERT_TRUE(manager.destroy_ptr(manager.find("named_obj").first)); ASSERT_TRUE( @@ -385,8 +377,8 @@ TEST(ManagerTest, DestroyDestruct) { // -- Check if destructors are called in destroy() -- // { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); int count = 3; auto *data_obj = manager.construct("named_obj")(); @@ -406,8 +398,8 @@ TEST(ManagerTest, DestroyDestruct) { TEST(ManagerTest, GetInstanceName) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_STREQ( manager.get_instance_name(manager.construct("named_obj")()), @@ -424,7 +416,7 @@ TEST(ManagerTest, GetInstanceName) { } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); ASSERT_STREQ( manager.get_instance_name(manager.find("named_obj").first), "named_obj"); @@ -453,9 +445,9 @@ TEST(ManagerTest, ConstructException) { } }; - manager_type::remove(dir_path().c_str()); + manager_type::remove(dir_path()); { - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); bool do_throw[2] = {false, true}; bool wrong_destroy = false; bool *flags[2] = {&wrong_destroy, &wrong_destroy}; @@ -477,9 +469,9 @@ TEST(ManagerTest, DestructException) { ~object() noexcept(false) { throw std::runtime_error(""); } }; - manager_type::remove(dir_path().c_str()); + manager_type::remove(dir_path()); { - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); manager.construct(metall::unique_instance)(); ASSERT_THROW(manager.destroy(metall::unique_instance), std::exception); @@ -488,8 +480,8 @@ TEST(ManagerTest, DestructException) { TEST(ManagerTest, GetInstanceType) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_EQ(manager.get_instance_kind(manager.construct("named_obj")()), metall::manager::instance_kind::named_kind); @@ -505,7 +497,7 @@ TEST(ManagerTest, GetInstanceType) { } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); ASSERT_EQ(manager.get_instance_kind(manager.find("named_obj").first), metall::manager::instance_kind::named_kind); @@ -523,8 +515,8 @@ TEST(ManagerTest, GetInstanceType) { TEST(ManagerTest, GetInstanceLength) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_EQ( manager.get_instance_length(manager.construct("named_obj")()), 1); @@ -553,7 +545,7 @@ TEST(ManagerTest, GetInstanceLength) { } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); { ASSERT_EQ( @@ -584,8 +576,8 @@ TEST(ManagerTest, GetInstanceLength) { TEST(ManagerTest, IsInstanceType) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_TRUE( manager.is_instance_type(manager.construct("named_obj")())); @@ -604,7 +596,7 @@ TEST(ManagerTest, IsInstanceType) { } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); ASSERT_TRUE( manager.is_instance_type(manager.find("named_obj").first)); @@ -624,8 +616,8 @@ TEST(ManagerTest, IsInstanceType) { TEST(ManagerTest, InstanceDescription) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); auto *named_obj = manager.construct("named_obj")(); std::string desc_name = "desc name"; @@ -651,7 +643,7 @@ TEST(ManagerTest, InstanceDescription) { } { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); std::string buf; @@ -677,7 +669,7 @@ TEST(ManagerTest, InstanceDescription) { } { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); std::string buf; @@ -706,8 +698,8 @@ TEST(ManagerTest, InstanceDescription) { TEST(ManagerTest, CountObjects) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_EQ(manager.get_num_named_objects(), 0); manager.construct("named_obj1")(); @@ -746,8 +738,8 @@ TEST(ManagerTest, CountObjects) { ptrdiff_t anony_offset1 = 0; ptrdiff_t anony_offset2 = 0; { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); manager.construct("named_obj1")(); manager.construct(metall::unique_instance)(); @@ -762,7 +754,7 @@ TEST(ManagerTest, CountObjects) { } { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); ASSERT_EQ(manager.get_num_named_objects(), 2); ASSERT_TRUE(manager.destroy("named_obj1")); @@ -788,8 +780,8 @@ TEST(ManagerTest, CountObjects) { TEST(ManagerTest, NamedObjectIterator) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); // Everyone is at the end ASSERT_EQ(manager.named_begin(), manager.named_end()); @@ -832,8 +824,8 @@ TEST(ManagerTest, NamedObjectIterator) { TEST(ManagerTest, UniqueObjectIterator) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); // Everyone is at the end ASSERT_EQ(manager.named_begin(), manager.named_end()); @@ -877,8 +869,8 @@ TEST(ManagerTest, UniqueObjectIterator) { TEST(ManagerTest, AnonymoustObjectIterator) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); // Everyone is at the end ASSERT_EQ(manager.named_begin(), manager.named_end()); @@ -924,9 +916,9 @@ TEST(ManagerTest, AnonymoustObjectIterator) { } TEST(ManagerTest, GetSegment) { - manager_type::remove(dir_path().c_str()); + manager_type::remove(dir_path()); { - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); auto *obj = manager.construct(metall::unique_instance)(); ASSERT_EQ(manager.unique_begin()->offset() + static_cast(manager.get_address()), @@ -935,46 +927,46 @@ TEST(ManagerTest, GetSegment) { } TEST(ManagerTest, Consistency) { - manager_type::remove(dir_path().c_str()); + manager_type::remove(dir_path()); { - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); // Must be inconsistent before closing - ASSERT_FALSE(manager_type::consistent(dir_path().c_str())); + ASSERT_FALSE(manager_type::consistent(dir_path())); manager.construct("dummy")(10); } - ASSERT_TRUE(manager_type::consistent(dir_path().c_str())); + ASSERT_TRUE(manager_type::consistent(dir_path())); { // To make sure the consistent mark is cleared even after creating a new // data store using an old dir path - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); - ASSERT_FALSE(manager_type::consistent(dir_path().c_str())); + ASSERT_FALSE(manager_type::consistent(dir_path())); manager.construct("dummy")(10); } - ASSERT_TRUE(manager_type::consistent(dir_path().c_str())); + ASSERT_TRUE(manager_type::consistent(dir_path())); { - manager_type manager(metall::open_only, dir_path().c_str()); - ASSERT_FALSE(manager_type::consistent(dir_path().c_str())); + manager_type manager(metall::open_only, dir_path()); + ASSERT_FALSE(manager_type::consistent(dir_path())); } - ASSERT_TRUE(manager_type::consistent(dir_path().c_str())); + ASSERT_TRUE(manager_type::consistent(dir_path())); { - manager_type manager(metall::open_read_only, dir_path().c_str()); + manager_type manager(metall::open_read_only, dir_path()); // Still consistent if it is opened with the read-only mode - ASSERT_TRUE(manager_type::consistent(dir_path().c_str())); + ASSERT_TRUE(manager_type::consistent(dir_path())); } - ASSERT_TRUE(manager_type::consistent(dir_path().c_str())); + ASSERT_TRUE(manager_type::consistent(dir_path())); } TEST(ManagerTest, TinyAllocation) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); const std::size_t alloc_size = k_min_object_size / 2; @@ -995,8 +987,8 @@ TEST(ManagerTest, TinyAllocation) { TEST(ManagerTest, SmallAllocation) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); const std::size_t alloc_size = k_min_object_size; @@ -1017,8 +1009,8 @@ TEST(ManagerTest, SmallAllocation) { TEST(ManagerTest, AllSmallAllocation) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); for (std::size_t s = 1; s < k_chunk_size; ++s) manager.deallocate(manager.allocate(s)); } @@ -1026,8 +1018,8 @@ TEST(ManagerTest, AllSmallAllocation) { TEST(ManagerTest, MaxSmallAllocation) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); // Max small allocation size const std::size_t alloc_size = @@ -1057,8 +1049,8 @@ TEST(ManagerTest, MaxSmallAllocation) { TEST(ManagerTest, MixedSmallAllocation) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str()); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path()); const std::size_t alloc_size1 = k_min_object_size * 2; const std::size_t alloc_size2 = k_min_object_size * 4; @@ -1097,8 +1089,8 @@ TEST(ManagerTest, MixedSmallAllocation) { TEST(ManagerTest, LargeAllocation) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str()); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path()); // Assume that the object cache is not used for large allocation char *base_addr = nullptr; @@ -1132,8 +1124,8 @@ TEST(ManagerTest, LargeAllocation) { TEST(ManagerTest, AllMemoryDeallocated) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path(), 1UL << 30UL); ASSERT_TRUE(manager.all_memory_deallocated()); @@ -1153,8 +1145,8 @@ TEST(ManagerTest, AllMemoryDeallocated) { TEST(ManagerTest, AlignedAllocation) { { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str()); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path()); for (std::size_t alignment = k_min_object_size; alignment <= k_chunk_size; alignment *= 2) { @@ -1208,21 +1200,20 @@ TEST(ManagerTest, AlignedAllocation) { } TEST(ManagerTest, Flush) { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str()); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path()); manager.construct("int")(10); manager.flush(); - ASSERT_FALSE(manager_type::consistent(dir_path().c_str())); + ASSERT_FALSE(manager_type::consistent(dir_path())); } TEST(ManagerTest, AnonymousConstruct) { - manager_type::remove(dir_path().c_str()); + manager_type::remove(dir_path()); manager_type *manager; - manager = - new manager_type(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager = new manager_type(metall::create_only, dir_path(), 1UL << 30UL); int *const a = manager->construct(metall::anonymous_instance)(); ASSERT_NE(a, nullptr); @@ -1238,10 +1229,9 @@ TEST(ManagerTest, AnonymousConstruct) { } TEST(ManagerTest, UniqueConstruct) { - manager_type::remove(dir_path().c_str()); + manager_type::remove(dir_path()); manager_type *manager; - manager = - new manager_type(metall::create_only, dir_path().c_str(), 1UL << 30UL); + manager = new manager_type(metall::create_only, dir_path(), 1UL << 30UL); int *const a = manager->construct(metall::unique_instance)(); ASSERT_NE(a, nullptr); @@ -1263,44 +1253,44 @@ TEST(ManagerTest, UniqueConstruct) { } TEST(ManagerTest, UUID) { - manager_type::remove(dir_path().c_str()); + manager_type::remove(dir_path()); std::string uuid; { - manager_type manager(metall::create_only, dir_path().c_str()); + manager_type manager(metall::create_only, dir_path()); - uuid = manager_type::get_uuid(dir_path().c_str()); + uuid = manager_type::get_uuid(dir_path()); ASSERT_FALSE(uuid.empty()); } { // Returns the same UUID? - manager_type manager(metall::open_only, dir_path().c_str()); - ASSERT_EQ(manager_type::get_uuid(dir_path().c_str()), uuid); + manager_type manager(metall::open_only, dir_path()); + ASSERT_EQ(manager_type::get_uuid(dir_path()), uuid); } { // Returns a new UUID? - manager_type manager(metall::create_only, dir_path().c_str()); - ASSERT_NE(manager_type::get_uuid(dir_path().c_str()), uuid); + manager_type manager(metall::create_only, dir_path()); + ASSERT_NE(manager_type::get_uuid(dir_path()), uuid); } } TEST(ManagerTest, Version) { - manager_type::remove(dir_path().c_str()); + manager_type::remove(dir_path()); { - manager_type manager(metall::create_only, dir_path().c_str()); - ASSERT_EQ(manager_type::get_version(dir_path().c_str()), METALL_VERSION); + manager_type manager(metall::create_only, dir_path()); + ASSERT_EQ(manager_type::get_version(dir_path()), METALL_VERSION); } { - manager_type manager(metall::open_only, dir_path().c_str()); - ASSERT_EQ(manager_type::get_version(dir_path().c_str()), METALL_VERSION); + manager_type manager(metall::open_only, dir_path()); + ASSERT_EQ(manager_type::get_version(dir_path()), METALL_VERSION); } } TEST(ManagerTest, Description) { // Set and get with non-static method { - manager_type::remove(dir_path().c_str()); - manager_type manager(metall::create_only, dir_path().c_str()); + manager_type::remove(dir_path()); + manager_type manager(metall::create_only, dir_path()); ASSERT_TRUE(manager.set_description("description1")); std::string description; @@ -1311,23 +1301,21 @@ TEST(ManagerTest, Description) { // Get with static method { std::string description; - ASSERT_TRUE( - manager_type::get_description(dir_path().c_str(), &description)); + ASSERT_TRUE(manager_type::get_description(dir_path(), &description)); ASSERT_STREQ(description.c_str(), "description1"); } // Set with static method { - manager_type::remove(dir_path().c_str()); + manager_type::remove(dir_path()); manager_type manager(metall::create_only, - dir_path().c_str()); // Make a new data store - ASSERT_TRUE( - manager_type::set_description(dir_path().c_str(), "description2")); + dir_path()); // Make a new data store + ASSERT_TRUE(manager_type::set_description(dir_path(), "description2")); } // Get with non-static method { - manager_type manager(metall::open_only, dir_path().c_str()); + manager_type manager(metall::open_only, dir_path()); std::string description; ASSERT_TRUE(manager.get_description(&description)); ASSERT_STREQ(description.c_str(), "description2"); @@ -1341,13 +1329,13 @@ TEST(ManagerTest, CheckSanity) { metall::logger::abort_on_critical_error(false); { - auto *manager = new manager_type(metall::create_only, dir_path().c_str()); + auto *manager = new manager_type(metall::create_only, dir_path()); ASSERT_TRUE(manager->check_sanity()); } { auto *bad_manager = - new manager_type(metall::open_only, (dir_path() + "-invalid").c_str()); + new manager_type(metall::open_only, dir_path().string() + "-invalid"); ASSERT_FALSE(bad_manager->check_sanity()); } } diff --git a/test/kernel/object_attribute_accessor_test.cpp b/test/kernel/object_attribute_accessor_test.cpp index e1f9b96f..335be02e 100644 --- a/test/kernel/object_attribute_accessor_test.cpp +++ b/test/kernel/object_attribute_accessor_test.cpp @@ -15,24 +15,24 @@ using namespace metall; auto attr_accessor_named() { return manager::access_named_object_attribute( - test_utility::make_test_path().c_str()); + test_utility::make_test_path()); } auto attr_accessor_unique() { return manager::access_unique_object_attribute( - test_utility::make_test_path().c_str()); + test_utility::make_test_path()); } auto attr_accessor_anonymous() { return manager::access_anonymous_object_attribute( - test_utility::make_test_path().c_str()); + test_utility::make_test_path()); } TEST(ObjectAttributeAccessorTest, Constructor) { - manager::remove(test_utility::make_test_path().c_str()); + manager::remove(test_utility::make_test_path()); { - manager mngr(create_only, test_utility::make_test_path().c_str(), + manager mngr(create_only, test_utility::make_test_path(), 1ULL << 30ULL); } @@ -42,10 +42,10 @@ TEST(ObjectAttributeAccessorTest, Constructor) { } TEST(ObjectAttributeAccessorTest, NumObjects) { - manager::remove(test_utility::make_test_path().c_str()); + manager::remove(test_utility::make_test_path()); { - manager mngr(create_only, test_utility::make_test_path().c_str(), + manager mngr(create_only, test_utility::make_test_path(), 1ULL << 30ULL); } @@ -56,7 +56,7 @@ TEST(ObjectAttributeAccessorTest, NumObjects) { } { - manager mngr(open_only, test_utility::make_test_path().c_str()); + manager mngr(open_only, test_utility::make_test_path()); mngr.construct("int1")(); mngr.construct("int2")(); mngr.construct(unique_instance)(); @@ -71,10 +71,10 @@ TEST(ObjectAttributeAccessorTest, NumObjects) { } TEST(ObjectAttributeAccessorTest, Count) { - manager::remove(test_utility::make_test_path().c_str()); + manager::remove(test_utility::make_test_path()); { - manager mngr(create_only, test_utility::make_test_path().c_str(), + manager mngr(create_only, test_utility::make_test_path(), 1ULL << 30ULL); } diff --git a/test/kernel/snapshot_test.cpp b/test/kernel/snapshot_test.cpp index 3f4b25cf..f0ad8175 100644 --- a/test/kernel/snapshot_test.cpp +++ b/test/kernel/snapshot_test.cpp @@ -6,6 +6,7 @@ #include "gtest/gtest.h" #include +#include #include @@ -13,42 +14,44 @@ namespace { -std::string original_dir_path() { - const std::string path(test_utility::make_test_path("original")); +namespace fs = std::filesystem; + +fs::path original_dir_path() { + const fs::path path(test_utility::make_test_path("original")); return path; } -std::string snapshot_dir_path() { - const std::string path(test_utility::make_test_path("snapshot")); +fs::path snapshot_dir_path() { + const fs::path path(test_utility::make_test_path("snapshot")); return path; } TEST(SnapshotTest, Snapshot) { - metall::manager::remove(original_dir_path().c_str()); - metall::manager::remove(snapshot_dir_path().c_str()); + metall::manager::remove(original_dir_path()); + metall::manager::remove(snapshot_dir_path()); { - metall::manager manager(metall::create_only, original_dir_path().c_str()); + metall::manager manager(metall::create_only, original_dir_path()); [[maybe_unused]] auto a = manager.construct("a")(1); [[maybe_unused]] auto b = manager.construct(metall::unique_instance)(2); - ASSERT_TRUE(manager.snapshot(snapshot_dir_path().c_str())); - ASSERT_TRUE(metall::manager::consistent(snapshot_dir_path().c_str())); + ASSERT_TRUE(manager.snapshot(snapshot_dir_path())); + ASSERT_TRUE(metall::manager::consistent(snapshot_dir_path())); // UUID const auto original_uuid = - metall::manager::get_uuid(original_dir_path().c_str()); + metall::manager::get_uuid(original_dir_path()); ASSERT_FALSE(original_uuid.empty()); const auto snapshot_uuid = - metall::manager::get_uuid(snapshot_dir_path().c_str()); + metall::manager::get_uuid(snapshot_dir_path()); ASSERT_FALSE(snapshot_uuid.empty()); ASSERT_NE(original_uuid, snapshot_uuid); // Version - ASSERT_EQ(metall::manager::get_version(original_dir_path().c_str()), - metall::manager::get_version(snapshot_dir_path().c_str())); + ASSERT_EQ(metall::manager::get_version(original_dir_path()), + metall::manager::get_version(snapshot_dir_path())); } { diff --git a/test/test_utility.hpp b/test/test_utility.hpp index fd656bd8..f8bae3b4 100644 --- a/test/test_utility.hpp +++ b/test/test_utility.hpp @@ -10,20 +10,26 @@ #include #include +#include_next +#include #include namespace test_utility { +namespace { +namespace fs = std::filesystem; +} + const char *k_test_dir_env_name = "METALL_TEST_DIR"; const char *k_default_test_dir = "/tmp/metall_test_dir"; namespace detail { -inline std::string get_test_dir() { +inline fs::path get_test_dir() { if (const char *env_p = std::getenv(k_test_dir_env_name)) { - return std::string(env_p); + return fs::path(env_p); } - return std::string(k_default_test_dir); + return fs::path(k_default_test_dir); } } // namespace detail @@ -33,11 +39,14 @@ inline bool create_test_dir() { return true; } -inline std::string make_test_path(const std::string &name = std::string()) { - return detail::get_test_dir() + "/metalltest" + "-" + - ::testing::UnitTest::GetInstance()->current_test_case()->name() + "-" + - ::testing::UnitTest::GetInstance()->current_test_info()->name() + "-" + - name; +inline fs::path make_test_path(const fs::path &name = fs::path()) { + std::stringstream file_name; + file_name << "metalltest-" + << ::testing::UnitTest::GetInstance()->current_test_case()->name() + << "-" + << ::testing::UnitTest::GetInstance()->current_test_info()->name() + << "-" << name.string(); + return detail::get_test_dir() / file_name.str(); } } // namespace test_utility diff --git a/tutorial/hands_on/README.md b/tutorial/hands_on/README.md index a520303b..58243064 100644 --- a/tutorial/hands_on/README.md +++ b/tutorial/hands_on/README.md @@ -117,22 +117,9 @@ g++ -std=c++17 [tutorial_program.cpp] -lstdc++fs -I../../include -I${BOOST_ROOT} Clang (or Apple clang) could be used instead of GCC to build Metall. However, please note that we haven't tested it intensively. - - -**On macOS >= 10.15 or Linux** +To run on macOS, Metall requires macOS >= 10.15. ```bash # Remove "-lstdc++fs" option clang++ -std=c++17 [tutorial_program.cpp] -I../../include -I${BOOST_ROOT} -``` - - -**On macOS < 10.15** - -The C++17 library is not available on macOS < 10.15. -One has to stop using C++17 library in Metall. -If METALL_DISABLE_CXX17_FILESYSTEM_LIB macro is defined, Metall uses its own file system operation implementation. - -```bash -clang++ -std=c++17 [tutorial_program.cpp] -I../../include -I${BOOST_ROOT} -DMETALL_DISABLE_CXX17_FILESYSTEM_LIB ``` \ No newline at end of file diff --git a/tutorial/nvmw21/README.md b/tutorial/nvmw21/README.md index df098202..c71ec21d 100644 --- a/tutorial/nvmw21/README.md +++ b/tutorial/nvmw21/README.md @@ -67,22 +67,9 @@ g++ -std=c++17 [tutorial_program.cpp] -lstdc++fs -I../../include -I${BOOST_ROOT} Clang (or Apple clang) could be used instead of GCC to build Metall. However, please note that we haven't tested it intensively. - - -**On macOS >= 10.15 or Linux** +To run on macOS, Metall requires macOS >= 10.15. ```bash # Remove "-lstdc++fs" option clang++ -std=c++17 [tutorial_program.cpp] -I../../include -I${BOOST_ROOT} -``` - - -**On macOS < 10.15** - -The C++17 library is not available on macOS < 10.15. -One has to stop using C++17 library in Metall. -If METALL_DISABLE_CXX17_FILESYSTEM_LIB macro is defined, Metall uses its own file system operation implementation. - -```bash -clang++ -std=c++17 [tutorial_program.cpp] -I../../include -I${BOOST_ROOT} -DMETALL_DISABLE_CXX17_FILESYSTEM_LIB ``` \ No newline at end of file diff --git a/verification/CMakeLists.txt b/verification/CMakeLists.txt index 66d65804..819839da 100644 --- a/verification/CMakeLists.txt +++ b/verification/CMakeLists.txt @@ -4,4 +4,5 @@ add_subdirectory(mmap) add_subdirectory(object_size) add_subdirectory(logger) add_subdirectory(soft_dirty) -add_subdirectory(file_io) \ No newline at end of file +add_subdirectory(file_io) +add_subdirectory(sparse_copy) \ No newline at end of file diff --git a/verification/sparse_copy/CMakeLists.txt b/verification/sparse_copy/CMakeLists.txt new file mode 100644 index 00000000..ac13e8f9 --- /dev/null +++ b/verification/sparse_copy/CMakeLists.txt @@ -0,0 +1 @@ +add_metall_executable(verify_sparse_copy verify_sparse_copy.cpp) \ No newline at end of file diff --git a/verification/sparse_copy/verify_sparse_copy.cpp b/verification/sparse_copy/verify_sparse_copy.cpp new file mode 100644 index 00000000..d69e45f5 --- /dev/null +++ b/verification/sparse_copy/verify_sparse_copy.cpp @@ -0,0 +1,87 @@ +// Copyright 2023 Lawrence Livermore National Security, LLC and other Metall +// Project Developers. See the top-level COPYRIGHT file for details. +// +// SPDX-License-Identifier: (Apache-2.0 OR MIT) + +#include +#include +#include + +#include +#include + +namespace mdtl = metall::mtlldetail; + +int main() { + const std::filesystem::path src_path = "source.dat"; + const ssize_t size = 1024 * 1024 * 32; + + if (!mdtl::create_file(src_path)) { + std::cerr << "Failed to create a file" << std::endl; + return EXIT_FAILURE; + } + + if (!mdtl::extend_file_size(src_path, size)) { + std::cerr << "Failed to extend file size" << std::endl; + return EXIT_FAILURE; + } + + { + auto [fd, map] = mdtl::map_file_write_mode(src_path, nullptr, size, 0); + if (!map) { + std::cerr << "Failed to map a file" << std::endl; + return EXIT_FAILURE; + } + auto* buf = reinterpret_cast(map); + buf[0] = 1; + buf[1024 * 1024 - 1] = 1; + if (!mdtl::munmap(fd, map, size, true)) { + std::cerr << "Failed to unmap a file" << std::endl; + return EXIT_FAILURE; + } + } + + if (mdtl::get_file_size(src_path) < size) { + std::cerr << "Failed to extend file size" << std::endl; + return EXIT_FAILURE; + } + + if (mdtl::get_actual_file_size(src_path) >= size) { + std::cerr << "Failed to create a sparse file" << std::endl; + std::cerr << "Actual file size: " << mdtl::get_actual_file_size(src_path) + << std::endl; + return EXIT_FAILURE; + } + + // Sparse copy + const std::filesystem::path dst_path = "destination.dat"; + if (!mdtl::fcpdtl::copy_file_sparse_manually(src_path, dst_path)) { + std::cerr << "Failed to copy a file" << std::endl; + return EXIT_FAILURE; + } + + std::cout << "Source file size: " << mdtl::get_file_size(src_path) + << std::endl; + std::cout << "Destination file size: " << mdtl::get_file_size(dst_path) + << std::endl; + + // Show actual size + std::cout << "Source actual file size: " + << mdtl::get_actual_file_size(src_path) << std::endl; + std::cout << "Destination actual file size: " + << mdtl::get_actual_file_size(dst_path) << std::endl; + + // if (mdtl::get_file_size(dst_path) < size) { + // std::cerr << "Failed to extend file size" << std::endl; + // return EXIT_FAILURE; + // } + +// if (mdtl::get_actual_file_size(dst_path) >= size) { +// std::cerr << "Failed to create a sparse file" << std::endl; +// std::cerr << "Actual file size: " << mdtl::get_actual_file_size(dst_path) +// << std::endl; +// return EXIT_FAILURE; +// } + + return 0; +} \ No newline at end of file