From 808d118289e8ea4a396fe53ab3f703695d17a690 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Wed, 13 Jul 2022 14:36:50 -0700 Subject: [PATCH 01/17] Bugfix: compare to ground truth file --- bench/adjacency_list/test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/adjacency_list/test/test.sh b/bench/adjacency_list/test/test.sh index 89a5e0b4..84b45bec 100644 --- a/bench/adjacency_list/test/test.sh +++ b/bench/adjacency_list/test/test.sh @@ -25,7 +25,7 @@ compare() { sort -k 1,1n -k2,2n < ${file1} > ${DATASTORE_DIR_ROOT}/file1_sorted check_program_exit_status - sort -k 1,1n -k2,2n < ${file1} > ${DATASTORE_DIR_ROOT}/file2_sorted + sort -k 1,1n -k2,2n < ${file2} > ${DATASTORE_DIR_ROOT}/file2_sorted check_program_exit_status echo "Compare the dumped edges" From ddd2f27d502abc24925f98b5ab6ca69ae29a2ebd Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Wed, 13 Jul 2022 14:51:39 -0700 Subject: [PATCH 02/17] Bugfix in adj-list simple test --- bench/adjacency_list/test/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bench/adjacency_list/test/test.sh b/bench/adjacency_list/test/test.sh index 84b45bec..dad39219 100644 --- a/bench/adjacency_list/test/test.sh +++ b/bench/adjacency_list/test/test.sh @@ -74,6 +74,7 @@ main() { echo "Create Test" ./run_adj_list_bench_metall -o ${DATASTORE_DIR_ROOT}/metall_test_dir -d ${DATASTORE_DIR_ROOT}/adj_out ${DATA}* check_program_exit_status + rm -f ${DATASTORE_DIR_ROOT}/adj_ref cat ${DATA}* >> ${DATASTORE_DIR_ROOT}/adj_ref compare "${DATASTORE_DIR_ROOT}/adj_out" "${DATASTORE_DIR_ROOT}/adj_ref" /bin/rm -f "${DATASTORE_DIR_ROOT}/adj_out" "${DATASTORE_DIR_ROOT}/adj_ref" From ac9e4bf0c0ad86dc85f2a9cc9822fbda67f7b304 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Wed, 13 Jul 2022 15:07:59 -0700 Subject: [PATCH 03/17] Bugfix in adj-list simple test --- bench/adjacency_list/test/test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/adjacency_list/test/test.sh b/bench/adjacency_list/test/test.sh index dad39219..09c328b6 100644 --- a/bench/adjacency_list/test/test.sh +++ b/bench/adjacency_list/test/test.sh @@ -29,7 +29,8 @@ compare() { check_program_exit_status echo "Compare the dumped edges" - diff ${DATASTORE_DIR_ROOT}/file1_sorted ${DATASTORE_DIR_ROOT}/file2_sorted > ${DATASTORE_DIR_ROOT}/file_diff + diff ${DATASTORE_DIR_ROOT}/file1_sorted ${DATASTORE_DIR_ROOT}/file2_sorted + #diff ${DATASTORE_DIR_ROOT}/file1_sorted ${DATASTORE_DIR_ROOT}/file2_sorted > ${DATASTORE_DIR_ROOT}/file_diff check_program_exit_status num_diff=$(< ${DATASTORE_DIR_ROOT}/file_diff wc -l) From af258b40052ec0503fcfadd9d756b25672fdc251 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Wed, 13 Jul 2022 15:26:00 -0700 Subject: [PATCH 04/17] Bugfix: use space instead of tab when dumping adj-list --- bench/adjacency_list/bench_driver.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/adjacency_list/bench_driver.hpp b/bench/adjacency_list/bench_driver.hpp index f56c8eda..dd72651d 100644 --- a/bench/adjacency_list/bench_driver.hpp +++ b/bench/adjacency_list/bench_driver.hpp @@ -297,7 +297,7 @@ inline void dump_adj_list(const adjacency_list_type &adj_list, const std::string for (auto key_itr = adj_list.keys_begin(), key_end = adj_list.keys_end(); key_itr != key_end; ++key_itr) { for (auto value_itr = adj_list.values_begin(key_itr->first), value_end = adj_list.values_end(key_itr->first); value_itr != value_end; ++value_itr) { - ofs << key_itr->first << "\t" << *value_itr << "\n"; + ofs << key_itr->first << " " << *value_itr << "\n"; } } ofs.close(); From b7a35a600ab6c8caf66ae06c3a70c33dfbf45cd8 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Wed, 13 Jul 2022 15:54:26 -0700 Subject: [PATCH 05/17] Bugfix: adj-list test --- bench/adjacency_list/test/test.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/bench/adjacency_list/test/test.sh b/bench/adjacency_list/test/test.sh index 09c328b6..20b5b697 100644 --- a/bench/adjacency_list/test/test.sh +++ b/bench/adjacency_list/test/test.sh @@ -29,8 +29,7 @@ compare() { check_program_exit_status echo "Compare the dumped edges" - diff ${DATASTORE_DIR_ROOT}/file1_sorted ${DATASTORE_DIR_ROOT}/file2_sorted - #diff ${DATASTORE_DIR_ROOT}/file1_sorted ${DATASTORE_DIR_ROOT}/file2_sorted > ${DATASTORE_DIR_ROOT}/file_diff + diff ${DATASTORE_DIR_ROOT}/file1_sorted ${DATASTORE_DIR_ROOT}/file2_sorted > ${DATASTORE_DIR_ROOT}/file_diff check_program_exit_status num_diff=$(< ${DATASTORE_DIR_ROOT}/file_diff wc -l) @@ -78,7 +77,7 @@ main() { rm -f ${DATASTORE_DIR_ROOT}/adj_ref cat ${DATA}* >> ${DATASTORE_DIR_ROOT}/adj_ref compare "${DATASTORE_DIR_ROOT}/adj_out" "${DATASTORE_DIR_ROOT}/adj_ref" - /bin/rm -f "${DATASTORE_DIR_ROOT}/adj_out" "${DATASTORE_DIR_ROOT}/adj_ref" + /bin/rm -f "${DATASTORE_DIR_ROOT}/adj_out" # Open the adj-list with the open mode and add more edges echo "" @@ -93,7 +92,6 @@ main() { echo "Open Test" ./test/open_metall -o ${DATASTORE_DIR_ROOT}/metall_test_dir -d ${DATASTORE_DIR_ROOT}/adj_out_open check_program_exit_status - cat ${DATA}* >> ${DATASTORE_DIR_ROOT}/adj_ref compare "${DATASTORE_DIR_ROOT}/adj_out_open" "${DATASTORE_DIR_ROOT}/adj_ref" # Open the adj-list and destroy it to test memory leak From 91cd65652d574601624928b9d51ddae06d4f4fc5 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Fri, 15 Jul 2022 20:42:22 -0700 Subject: [PATCH 06/17] Add JSON container test cases. --- test/container/json/json_value.cpp | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/container/json/json_value.cpp b/test/container/json/json_value.cpp index 877b2de6..09c5bd8b 100644 --- a/test/container/json/json_value.cpp +++ b/test/container/json/json_value.cpp @@ -278,4 +278,40 @@ TEST (JSONValueTest, Equal) { jv1.as_object()["object"].as_object()["currency"].as_string() = "JPY"; GTEST_ASSERT_NE(jv1, jv2); } + +TEST (JSONValueTest, EqualBool) { + auto jv = json::value(); + jv.emplace_bool() = true; + GTEST_ASSERT_EQ(jv, true); + GTEST_ASSERT_NE(jv, -10); + GTEST_ASSERT_NE(jv, 10); + GTEST_ASSERT_NE(jv, 10.0); +} + +TEST (JSONValueTest, EqualInt64) { + auto jv = json::value(); + jv.emplace_int64() = -10; + GTEST_ASSERT_NE(jv, true); + GTEST_ASSERT_EQ(jv, -10); + GTEST_ASSERT_NE(jv, 10); + GTEST_ASSERT_NE(jv, 10.0); +} + +TEST (JSONValueTest, EqualUint64) { + auto jv = json::value(); + jv.emplace_uint64() = 10; + GTEST_ASSERT_NE(jv, true); + GTEST_ASSERT_NE(jv, -10); + GTEST_ASSERT_EQ(jv, 10); + GTEST_ASSERT_NE(jv, 10.0); +} + +TEST (JSONValueTest, EqualDouble) { + auto jv = json::value(); + jv.emplace_double() = 10.0; + GTEST_ASSERT_NE(jv, true); + GTEST_ASSERT_NE(jv, -10); + GTEST_ASSERT_NE(jv, 10); + GTEST_ASSERT_EQ(jv, 10.0); +} } \ No newline at end of file From 18c273d132d3d8b58d1eb8cb20c19a54ba2b1cd2 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Mon, 29 Aug 2022 16:47:38 -0700 Subject: [PATCH 07/17] Fix a link error --- example/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/README.md b/example/README.md index 669e9bb0..10912a71 100644 --- a/example/README.md +++ b/example/README.md @@ -1,6 +1,6 @@ # List of Examples -To build examples see a [page](https://metall.readthedocs.io/en/latest/advanced_build/example_test_bench/) hosted on Read the Docs. +To build examples see a [page](https://metall.readthedocs.io/en/latest/advanced_build/cmake/) hosted on Read the Docs. ### Basic examples From 2f4fede1f0495bb091651b940e69f4a10b74a284 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Mon, 29 Aug 2022 16:47:53 -0700 Subject: [PATCH 08/17] Code brush up --- bench/adjacency_list/test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench/adjacency_list/test/test.sh b/bench/adjacency_list/test/test.sh index 20b5b697..86590266 100644 --- a/bench/adjacency_list/test/test.sh +++ b/bench/adjacency_list/test/test.sh @@ -18,7 +18,7 @@ compare() { num_elements=$(< ${file1} wc -l) if [[ ${num_elements} -eq 0 ]]; then echo "<< ${file1} is empty!! >>" - exit + exit 1 fi echo "Sort the dumped edges" From 70974ffe86bfe9be59e84e2fd44a94d31133e530 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Tue, 13 Sep 2022 09:24:41 -0700 Subject: [PATCH 09/17] [skip ci] Update Read the Docs files. --- docs/readthedocs/about/publication.md | 31 ++++++++++++------- .../basics/compile_time_options.md | 6 ++-- docs/readthedocs/detail/api.md | 8 +++-- docs/readthedocs/detail/example.md | 2 +- .../detail/internal_architecture.md | 2 +- docs/readthedocs/index.md | 20 +++--------- mkdocs.yml | 1 + 7 files changed, 35 insertions(+), 35 deletions(-) diff --git a/docs/readthedocs/about/publication.md b/docs/readthedocs/about/publication.md index b8412f3b..e1b8e0e8 100644 --- a/docs/readthedocs/about/publication.md +++ b/docs/readthedocs/about/publication.md @@ -1,17 +1,26 @@ # Publication -## Metall: A Persistent Memory Allocator Enabling Graph Processing +## Metall: A persistent memory allocator for data-centric analytics + +``` +Keita Iwabuchi, Karim Youssef, Kaushik Velusamy, Maya Gokhale, Roger Pearce, +Metall: A persistent memory allocator for data-centric analytics, +Parallel Computing, 2022, 102905, ISSN 0167-8191, https://doi.org/10.1016/j.parco.2022.102905. +``` + +- [Parallel Computing](https://www.sciencedirect.com/science/article/abs/pii/S0167819122000114) (journal) -[Available PDF](https://www.osti.gov/servlets/purl/1576900) +- [arXiv](https://arxiv.org/abs/2108.07223) (preprint) + +## Metall: A Persistent Memory Allocator Enabling Graph Processing -A page in IEEE Xplore is [here](https://ieeexplore.ieee.org/document/8945094) ```text -@INPROCEEDINGS{8945094, -author={K. {Iwabuchi} and L. {Lebanoff} and M. {Gokhale} and R. {Pearce}}, -booktitle={2019 IEEE/ACM 9th Workshop on Irregular Applications: Architectures and Algorithms (IA3)}, -title={Metall: A Persistent Memory Allocator Enabling Graph Processing}, -year={2019}, -pages={39-44}, -doi={10.1109/IA349570.2019.00012}, -month={Nov},} +K. Iwabuchi, L. Lebanoff, M. Gokhale and R. Pearce, +"Metall: A Persistent Memory Allocator Enabling Graph Processing," +2019 IEEE/ACM 9th Workshop on Irregular Applications: Architectures and Algorithms (IA3), 2019, +pp. 39-44, doi: 10.1109/IA349570.2019.00012. ``` + +- [Available PDF](https://www.osti.gov/servlets/purl/1576900) + +- A page in IEEE Xplore is [here](https://ieeexplore.ieee.org/document/8945094) diff --git a/docs/readthedocs/basics/compile_time_options.md b/docs/readthedocs/basics/compile_time_options.md index 609337cc..f30fdc98 100644 --- a/docs/readthedocs/basics/compile_time_options.md +++ b/docs/readthedocs/basics/compile_time_options.md @@ -1,6 +1,6 @@ # Compile-time Options -There are some compile-time options (macro) as follows to configure the behavior of Metall: +There are some compile-time options (C/C++ macro) as follows to configure the behavior of Metall: - METALL_DISABLE_FREE_FILE_SPACE @@ -10,13 +10,13 @@ There are some compile-time options (macro) as follows to configure the behavior - METALL_DEFAULT_VM_RESERVE_SIZE=*bytes* - The default virtual memory reserve size - An internally defined value is used if 0 is specified - - Wll be rounded up to a multiple of the page size internally + - Wll be rounded up to a multiple of the system page size (e.g., 4 KB) internally - METALL_INITIAL_SEGMENT_SIZE=*bytes* - The initial segment size - Use the internally defined value if 0 is specified - - Wll be rounded up to a multiple of the page size internally + - Wll be rounded up to a multiple of the system page size internally - METALL_FREE_SMALL_OBJECT_SIZE_HINT=*bytes* diff --git a/docs/readthedocs/detail/api.md b/docs/readthedocs/detail/api.md index c7e8f071..4caddd88 100644 --- a/docs/readthedocs/detail/api.md +++ b/docs/readthedocs/detail/api.md @@ -13,6 +13,8 @@ In multi-process environment, Metall assumes each process allocates its own Meta ## Main APIs in Metall +Here, we list Metall's main APIs. + ```C++ // The main header file #include @@ -118,11 +120,11 @@ static bool metall::get_description(const char *dir_path, std::string *descripti Example programs are located in [example](https://github.com/LLNL/metall/tree/master/example). -## Generate API document using Doxygen +## FUll API document -A Doxygen configuration file is [here](https://github.com/LLNL/metall/tree/master/docs/Doxyfile.in). +The full API document is available [here](https://software.llnl.gov/metall/api/). -To generate API document: +To generate the full API document locally using Doxygen: ```bash cd metall diff --git a/docs/readthedocs/detail/example.md b/docs/readthedocs/detail/example.md index 93e66d35..d5d82914 100644 --- a/docs/readthedocs/detail/example.md +++ b/docs/readthedocs/detail/example.md @@ -2,4 +2,4 @@ Example programs are located in [example](https://github.com/LLNL/metall/tree/master/example) -To build the examples see [build source files in Metall](../advanced_build/example_test_bench.md) \ No newline at end of file +To build the examples see [build source files in Metall](../advanced_build/cmake.md) \ No newline at end of file diff --git a/docs/readthedocs/detail/internal_architecture.md b/docs/readthedocs/detail/internal_architecture.md index b12143a6..da73bc48 100644 --- a/docs/readthedocs/detail/internal_architecture.md +++ b/docs/readthedocs/detail/internal_architecture.md @@ -26,7 +26,7 @@ Objects larger than the half chunk size (*large objects*) use a single or multip By default, Metall frees DRAM and file space by chunk, that is, small object deallocations do not free physical memory immediately, whereas large object deallocations do. Metall also has a mode that tries to free the corresponding space when an object equal or larger than *N* bytes is deallocated, -where N is set by the compile time option *METALL_FREE_SMALL_OBJECT_SIZE_HINT=N* (see [Build and Install](../getting_started/build_and_install.md)). +where N is set by the compile time option (macro) *METALL_FREE_SMALL_OBJECT_SIZE_HINT=N* (see [Build and Install](../basics/compile_time_options.md)). ### Internal Allocation Size Same as other major heap memory allocators, Metall rounds up a small object to the nearest internal allocation size. diff --git a/docs/readthedocs/index.md b/docs/readthedocs/index.md index f465ab91..df98b3fa 100644 --- a/docs/readthedocs/index.md +++ b/docs/readthedocs/index.md @@ -1,3 +1,7 @@ +# Metall: Persistent Memory Allocator for Data-Centric Analytics + +This Read the Docs page describes an open-source persistent memory allocator Metall, available at [here]((https://github.com/LLNL/metall)). + ## Overview Metall is a persistent memory allocator designed to provide developers with an API to allocate custom C++ data structures in both block-storage and @@ -18,20 +22,4 @@ Example programs that use Metall are listed [here](detail/example.md). ![Metall Overview](./img/metall_overview.png) - ## Publication - -[Metall: A Persistent Memory Allocator Enabling Graph Processing](https://www.osti.gov/servlets/purl/1576900) - -```text -@INPROCEEDINGS{8945094, -author={K. {Iwabuchi} and L. {Lebanoff} and M. {Gokhale} and R. {Pearce}}, -booktitle={2019 IEEE/ACM 9th Workshop on Irregular Applications: Architectures and Algorithms (IA3)}, -title={Metall: A Persistent Memory Allocator Enabling Graph Processing}, -year={2019}, -pages={39-44}, -doi={10.1109/IA349570.2019.00012}, -month={Nov},} -``` - -IEEE Xplore [page](https://ieeexplore.ieee.org/document/8945094) \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 7af06802..84a8d328 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -30,4 +30,5 @@ nav: - 'Example': 'detail/example.md' - 'Internal Architecture': 'detail/internal_architecture.md' - 'ABOUT': + - 'Publication': 'about/publication.md' - 'License and Notice': 'about/license_notice.md' From 73a25ee20a22715f7514339a6c888841c3d5405c Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Tue, 13 Sep 2022 09:30:16 -0700 Subject: [PATCH 10/17] [skip ci] Update Read the Docs files. --- docs/readthedocs/index.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/readthedocs/index.md b/docs/readthedocs/index.md index df98b3fa..95b9135e 100644 --- a/docs/readthedocs/index.md +++ b/docs/readthedocs/index.md @@ -1,6 +1,6 @@ # Metall: Persistent Memory Allocator for Data-Centric Analytics -This Read the Docs page describes an open-source persistent memory allocator Metall, available at [here]((https://github.com/LLNL/metall)). +This Read the Docs page describes Metall (open-source library available [here]((https://github.com/LLNL/metall))). ## Overview @@ -21,5 +21,3 @@ and provides persistent memory snapshotting (versioning) capabilities. Example programs that use Metall are listed [here](detail/example.md). ![Metall Overview](./img/metall_overview.png) - -## Publication From 04e8448bb5fb60327dac4bde266d106bfc8ec6cb Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Mon, 3 Oct 2022 14:16:34 -0700 Subject: [PATCH 11/17] Bugfix: anonymous new mapping --- .../segment_storage/mmap_segment_storage.hpp | 153 +++++++++++++----- 1 file changed, 117 insertions(+), 36 deletions(-) diff --git a/include/metall/kernel/segment_storage/mmap_segment_storage.hpp b/include/metall/kernel/segment_storage/mmap_segment_storage.hpp index d53d0b03..ac86fbc4 100644 --- a/include/metall/kernel/segment_storage/mmap_segment_storage.hpp +++ b/include/metall/kernel/segment_storage/mmap_segment_storage.hpp @@ -36,8 +36,6 @@ class mmap_segment_storage { // -------------------------------------------------------------------------------- // mmap_segment_storage() { #ifdef METALL_USE_ANONYMOUS_NEW_MAP - // TODO: implement msync for anonymous mapping - static_assert(true, "METALL_USE_ANONYMOUS_NEW_MAP does not work now"); logger::out(logger::level::info, __FILE__, __LINE__, "METALL_USE_ANONYMOUS_NEW_MAP is defined"); #endif @@ -73,7 +71,11 @@ class mmap_segment_storage { m_read_only(other.m_read_only), m_free_file_space(other.m_free_file_space), m_block_fd_list(std::move(other.m_block_fd_list)), - m_block_size(other.m_block_size) { + m_block_size(other.m_block_size) +#ifdef METALL_USE_ANONYMOUS_NEW_MAP + , m_anonymous_map_flag_list(other.m_anonymous_map_flag_list) +#endif + { other.priv_set_broken_status(); } @@ -88,7 +90,9 @@ class mmap_segment_storage { m_free_file_space = other.m_free_file_space; m_block_fd_list = std::move(other.m_block_fd_list); m_block_size = other.m_block_size; - +#ifdef METALL_USE_ANONYMOUS_NEW_MAP + m_anonymous_map_flag_list = std::move(other.m_anonymous_map_flag_list); +#endif other.priv_set_broken_status(); return (*this); } @@ -254,7 +258,8 @@ class mmap_segment_storage { } bool priv_is_open() const { - return (check_sanity() && m_system_page_size > 0 && m_num_blocks > 0 && m_vm_region_size > 0 && m_current_segment_size > 0 + return (check_sanity() && m_system_page_size > 0 && m_num_blocks > 0 && m_vm_region_size > 0 + && m_current_segment_size > 0 && m_segment && !m_base_path.empty() && !m_block_fd_list.empty() && m_block_size > 0); } @@ -329,7 +334,10 @@ class mmap_segment_storage { return true; } - bool priv_open(const std::string &base_path, const size_type vm_region_size, void *const vm_region, const bool read_only) { + bool priv_open(const std::string &base_path, + const size_type vm_region_size, + void *const vm_region, + const bool read_only) { if (!check_sanity()) return false; if (is_open()) return false; // Cannot open multiple segments simultaneously. @@ -362,14 +370,16 @@ class mmap_segment_storage { const auto fd = priv_map_file(file_name, m_block_size, m_current_segment_size, read_only); if (fd == -1) { std::stringstream ss; - ss << "Failed to map a file " << m_block_size; + ss << "Failed to map a file " << file_name; logger::out(logger::level::error, __FILE__, __LINE__, ss.str().c_str()); priv_destroy_segment(); priv_set_broken_status(); return false; - } else { - m_block_fd_list.template emplace_back(fd); } + m_block_fd_list.emplace_back(fd); +#ifdef METALL_USE_ANONYMOUS_NEW_MAP + m_anonymous_map_flag_list.push_back(false); +#endif m_current_segment_size += m_block_size; ++m_num_blocks; } @@ -445,27 +455,21 @@ class mmap_segment_storage { } #ifdef METALL_USE_ANONYMOUS_NEW_MAP - if (!priv_map_anonymous(file_name, file_size, segment_offset)) { - return false; + const auto fd = priv_map_anonymous(file_name, file_size, segment_offset); + if (m_anonymous_map_flag_list.size() < block_number + 1) { + m_anonymous_map_flag_list.resize(block_number + 1, false); } - // Although we do not map the file, we still open it so that other functions in this class work. - const auto fd = ::open(file_name.c_str(), O_RDWR); - if (fd == -1) { - logger::perror(logger::level::error, __FILE__, __LINE__, "open"); - std::string s("Failed to open a file " + file_name); - logger::out(logger::level::error, __FILE__, __LINE__, s.c_str()); - // Destroy the map by overwriting PROT_NONE map since the VM region is managed by another class. - mdtl::map_with_prot_none(static_cast(m_segment) + segment_offset, file_size); - return false; - } - m_block_fd_list.emplace_back(fd); + m_anonymous_map_flag_list[block_number] = true; #else const auto fd = priv_map_file(file_name, file_size, segment_offset, false); +#endif if (fd == -1) { return false; } - m_block_fd_list.emplace_back(fd); -#endif + if (m_block_fd_list.size() < block_number + 1) { + m_block_fd_list.resize(block_number + 1, -1); + } + m_block_fd_list[block_number] = fd; return true; } @@ -510,9 +514,9 @@ class mmap_segment_storage { return ret.first; } - bool priv_map_anonymous(const std::string &path, - const size_type region_size, - const different_type segment_offset) const { + int priv_map_anonymous(const std::string &path, + const size_type region_size, + const different_type segment_offset) const { assert(!path.empty()); assert(region_size > 0); assert(segment_offset >= 0); @@ -529,17 +533,28 @@ class mmap_segment_storage { if (!addr) { std::string s("Failed to map an anonymous region at " + std::to_string(segment_offset)); logger::out(logger::level::error, __FILE__, __LINE__, s.c_str()); - return false; + return -1; } - return true; + // Although we do not map the file, we still open it so that other functions in this class works. + const auto fd = ::open(path.c_str(), O_RDWR); + if (fd == -1) { + logger::perror(logger::level::error, __FILE__, __LINE__, "open"); + std::string s("Failed to open a file " + path); + logger::out(logger::level::error, __FILE__, __LINE__, s.c_str()); + // Destroy the map by overwriting PROT_NONE map since the VM region is managed by another class. + mdtl::map_with_prot_none(static_cast(m_segment) + segment_offset, region_size); + return -1; + } + + return fd; } bool priv_destroy_segment() { if (!is_open()) return false; int succeeded = true; - for (const auto &fd : m_block_fd_list) { + for (const auto &fd: m_block_fd_list) { succeeded &= mdtl::os_close(fd); } @@ -553,7 +568,7 @@ class mmap_segment_storage { return succeeded; } - bool priv_sync(const bool sync) const { + bool priv_sync(const bool sync) { if (!priv_sync_segment(sync)) { // Failing this operation is not a critical error logger::out(logger::level::error, __FILE__, __LINE__, "Failed to synchronize the segment"); return false; @@ -561,7 +576,7 @@ class mmap_segment_storage { return true; } - bool priv_sync_segment(const bool sync) const { + bool priv_sync_segment(const bool sync) { if (!is_open()) return false; if (m_read_only) return true; @@ -591,7 +606,7 @@ class mmap_segment_storage { return true; } - bool priv_parallel_msync(const bool sync) const { + bool priv_parallel_msync(const bool sync) { std::atomic_uint_fast64_t block_no_count = 0; std::atomic_uint_fast64_t num_successes = 0; @@ -599,6 +614,13 @@ class mmap_segment_storage { while (true) { const auto block_no = block_no_count.fetch_add(1); if (block_no < m_block_fd_list.size()) { +#ifdef METALL_USE_ANONYMOUS_NEW_MAP + assert(m_anonymous_map_flag_list.size() > block_no); + if (m_anonymous_map_flag_list[block_no]) { + num_successes.fetch_add(priv_sync_anonymous_map(block_no) ? 1 : 0); + continue; + } +#endif const auto map = static_cast(m_segment) + block_no * m_block_size; num_successes.fetch_add(mdtl::os_msync(map, m_block_size, sync) ? 1 : 0); } else { @@ -614,11 +636,11 @@ class mmap_segment_storage { logger::out(logger::level::info, __FILE__, __LINE__, ss.str().c_str()); } std::vector> threads(num_threads); - for (auto &th : threads) { - th = std::make_unique(diff_sync); + for (auto &th: threads) { + th = std::make_unique(diff_sync); } - for (auto &th : threads) { + for (auto &th: threads) { th->join(); } @@ -630,6 +652,14 @@ class mmap_segment_storage { if (offset + nbytes > m_current_segment_size) return false; +#ifdef METALL_USE_ANONYMOUS_NEW_MAP + const auto block_no = offset / m_block_size; + assert(m_anonymous_map_flag_list.size() > block_no); + if (m_anonymous_map_flag_list[block_no]) { + return priv_uncommit_private_anonymous_pages(offset, nbytes); + } +#endif + if (m_free_file_space) return priv_uncommit_pages_and_free_file_space(offset, nbytes); else @@ -644,6 +674,54 @@ class mmap_segment_storage { return mdtl::uncommit_shared_pages(static_cast(m_segment) + offset, nbytes); } + bool priv_uncommit_private_anonymous_pages(const different_type offset, const size_type nbytes) const { + return mdtl::uncommit_private_anonymous_pages(static_cast(m_segment) + offset, nbytes); + } + +#ifdef METALL_USE_ANONYMOUS_NEW_MAP + bool priv_sync_anonymous_map(const size_type block_no) { + assert(m_anonymous_map_flag_list[block_no]); + { + std::string s("Sync anonymous map at block " + std::to_string(block_no)); + logger::out(logger::level::info, __FILE__, __LINE__, s.c_str()); + } + + auto *const addr = static_cast(m_segment) + block_no * m_block_size; + if (::write(m_block_fd_list[block_no], addr, m_block_size) != (ssize_t)m_block_size) { + std::string s("Failed to write back a block"); + logger::perror(logger::level::error, __FILE__, __LINE__, s.c_str()); + priv_destroy_segment(); + priv_set_broken_status(); + return false; + } + m_anonymous_map_flag_list[block_no] = false; + + { + std::string s("Map block " + std::to_string(block_no) + " as a non-anonymous map"); + logger::out(logger::level::info, __FILE__, __LINE__, s.c_str()); + } + [[maybe_unused]] static constexpr int map_nosync = +#ifdef MAP_NOSYNC + MAP_NOSYNC; +#else + 0; +#endif + const auto mapped_addr = mdtl::map_file_write_mode(m_block_fd_list[block_no], + addr, + m_block_size, + 0, + MAP_FIXED | map_nosync); + if (!mapped_addr || mapped_addr != addr) { + std::string s("Failed to map a block"); + logger::out(logger::level::error, __FILE__, __LINE__, s.c_str()); + priv_destroy_segment(); + priv_set_broken_status(); + return false; + } + return true; + } +#endif + bool priv_set_system_page_size() { m_system_page_size = mdtl::get_page_size(); if (m_system_page_size == -1) { @@ -721,6 +799,9 @@ class mmap_segment_storage { std::vector m_block_fd_list; size_type m_block_size{0}; bool m_broken{false}; +#ifdef METALL_USE_ANONYMOUS_NEW_MAP + std::vector m_anonymous_map_flag_list; +#endif }; } // namespace kernel From 7cf0fbd7f02edd4087d1958db0c52e5317256971 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Mon, 3 Oct 2022 14:43:46 -0700 Subject: [PATCH 12/17] Add mapping bench --- bench/CMakeLists.txt | 3 +- bench/mapping/CMakeLists.txt | 1 + bench/mapping/run_mapping_bench.cpp | 206 ++++++++++++++++++++++++++++ 3 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 bench/mapping/CMakeLists.txt create mode 100644 bench/mapping/run_mapping_bench.cpp diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt index d5c6201b..123cc662 100644 --- a/bench/CMakeLists.txt +++ b/bench/CMakeLists.txt @@ -1,4 +1,5 @@ add_subdirectory(simple_alloc) add_subdirectory(adjacency_list) add_subdirectory(bfs) -add_subdirectory(rand_engine) \ No newline at end of file +add_subdirectory(rand_engine) +add_subdirectory(mapping) \ No newline at end of file diff --git a/bench/mapping/CMakeLists.txt b/bench/mapping/CMakeLists.txt new file mode 100644 index 00000000..e79eb906 --- /dev/null +++ b/bench/mapping/CMakeLists.txt @@ -0,0 +1 @@ +add_metall_executable(run_mapping_bench run_mapping_bench.cpp) \ No newline at end of file diff --git a/bench/mapping/run_mapping_bench.cpp b/bench/mapping/run_mapping_bench.cpp new file mode 100644 index 00000000..04a06d32 --- /dev/null +++ b/bench/mapping/run_mapping_bench.cpp @@ -0,0 +1,206 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using rand_engine = metall::utility::rand_512; +static constexpr std::size_t k_page_size = 4096; + +namespace { +namespace mdtl = metall::mtlldetail; +} + +auto random_write_by_page(const std::size_t size, unsigned char *const map) { + const auto num_pages = size / k_page_size; + rand_engine rand_engine(123); + std::uniform_int_distribution<> dist(0, num_pages - 1); + + const auto s = mdtl::elapsed_time_sec(); + for (std::size_t i = 0; i < num_pages; ++i) { + const auto page_no = dist(rand_engine); + const off_t offset = static_cast(page_no * k_page_size); + map[offset] = '0'; + } + const auto t = mdtl::elapsed_time_sec(s); + + return t; +} + +auto random_read_by_page(const std::size_t size, const unsigned char *const map) { + const auto num_pages = size / k_page_size; + rand_engine rand_engine(1234); + std::uniform_int_distribution<> dist(0, num_pages - 1); + + const auto s = mdtl::elapsed_time_sec(); + for (std::size_t i = 0; i < num_pages; ++i) { + const auto page_no = dist(rand_engine); + const off_t offset = static_cast(page_no * k_page_size); + [[maybe_unused]] volatile char dummy = map[offset]; + } + const auto t = mdtl::elapsed_time_sec(s); + + return t; +} + +int create_normal_file(std::string_view path) { + const int fd = ::open(path.data(), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR); + if (fd == -1) { + std::cerr << "Failed to create a file" << std::endl; + std::abort(); + } + return fd; +} + +int create_tmpfile(std::string_view path) { + static char file_template[] = "/mmap.XXXXXX"; + + char fullname[path.size() + sizeof(file_template)]; + (void)strcpy(fullname, path.data()); + (void)strcat(fullname, file_template); + + int fd = -1; + if ((fd = mkstemp(fullname)) < 0) { + std::perror("Could not create temporary file"); + std::abort(); + } + + (void)unlink(fullname); + + return fd; +} + +void extend_file(const int fd, const std::size_t size, const bool fill_with_zero) { + if (!mdtl::extend_file_size(fd, size, fill_with_zero)) { + std::cerr << "Failed to extend file" << std::endl; + std::abort(); + } +} + +auto map_file(const int fd, const std::size_t size) { + static constexpr int k_map_nosync = +#ifdef MAP_NOSYNC + MAP_NOSYNC; +#else + 0; +#warning "MAP_NOSYNC is not defined" +#endif + + auto *const map = mdtl::os_mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED | k_map_nosync, fd, 0); + if (!map) { + std::cerr << " Failed mapping" << std::endl; + std::abort(); + } + + return map; +} + +void close_file(const int fd) { + if (!mdtl::os_close(fd)) { + std::cerr << __LINE__ << " Failed to close file" << std::endl; + std::abort(); + } +} + +void unmap(void *const addr, const std::size_t size) { + if (!mdtl::munmap(addr, size, false)) { + std::cerr << __LINE__ << " Failed to munmap" << std::endl; + std::abort(); + } +} + +/// \brief Run benchmark to evaluate different mapping methods +void run_bench_one_time(std::string_view dir_path, + const std::size_t length, + const bool init_file_writing_zero, + std::map> &time_table) { + + const auto bench_core = [&time_table, length](std::string_view mode, unsigned char *const map) { + time_table[std::string(mode) + " write"].push_back(random_write_by_page(length, map)); + time_table[std::string(mode) + " read"].push_back(random_read_by_page(length, map)); + }; + + // Use 'new' + { + auto *const map = new unsigned char[length]; + bench_core("malloc", map); + delete[] map; + } + + // Use a normal file and mmap + { + std::string file_path{std::string(dir_path) + "/map-file"}; + const int fd = create_normal_file(file_path); + extend_file(fd, length, init_file_writing_zero); + auto *const map = static_cast(map_file(fd, length)); + close_file(fd); + bench_core("Normal-file", map); + unmap(map, length); + } + + // Use tmpfile and mmap + { + const int fd = create_tmpfile(dir_path); + extend_file(fd, length, init_file_writing_zero); + auto *const map = static_cast(map_file(fd, length)); + close_file(fd); + bench_core("tmpfile", map); + unmap(map, length); + } + + // Use Metall + { + metall::manager manager(metall::create_only, dir_path.data()); + auto *map = static_cast(manager.allocate(length)); + bench_core("Metall", map); + manager.deallocate(map); + } + metall::manager::remove(dir_path.data()); + +} + +void run_bench(std::string_view dir_path, + const std::size_t num_repeats, + const std::size_t length, + const bool init_file_writing_zero) { + std::cout << "\n----------" << std::endl; + std::cout << "Directory Path:\t" << dir_path + << "\nRepeats:\t" << num_repeats + << "\nLength:\t" << length + << "\nInit w/ writing:\t" << init_file_writing_zero + << "\n" << std::endl; + + // Run bench + std::map> time_table; + for (std::size_t i = 0; i < num_repeats; ++i) { + run_bench_one_time(dir_path, length, init_file_writing_zero, time_table); + } + + // Show results + for (const auto &entry: time_table) { + const auto &mode = entry.first; + const auto × = entry.second; + std::cout << std::fixed; + std::cout << std::setprecision(2); + std::cout << mode << " took (s)\t" + << std::accumulate(times.begin(), times.end(), 0.0f) / times.size() << std::endl; + } +} + +int main() { + static constexpr std::size_t size = k_page_size * 1024 * 10; + const int num_repeats = 10; + +#if defined(__linux__) + run_bench("/dev/shm", num_repeats, size, false); + run_bench("/dev/shm", num_repeats, size, true); +#endif + run_bench("/tmp", num_repeats, size, false); + run_bench("/tmp", num_repeats, size, true); + + return 0; +} \ No newline at end of file From c3029824c892b2d5dd78d1d57997c6abde750158 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Tue, 4 Oct 2022 16:42:05 -0700 Subject: [PATCH 13/17] Bugfix and speed up in all_memory_deallocated --- include/metall/kernel/chunk_directory.hpp | 9 ++++----- include/metall/kernel/segment_allocator.hpp | 15 +++++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/include/metall/kernel/chunk_directory.hpp b/include/metall/kernel/chunk_directory.hpp index 2f642504..30b67c1e 100644 --- a/include/metall/kernel/chunk_directory.hpp +++ b/include/metall/kernel/chunk_directory.hpp @@ -412,17 +412,16 @@ class chunk_directory { return buf; } - std::vector get_all_large_chunks() const { - std::vector buf; - + const std::size_t num_used_large_chunks() const { + std::size_t count = 0; for (chunk_no_type chunk_no = 0; chunk_no < size(); ++chunk_no) { if (m_table[chunk_no].type == chunk_type::large_chunk_head || m_table[chunk_no].type == chunk_type::large_chunk_body) { - buf.push_back(chunk_no); + ++count; } } - return buf; + return count; } private: diff --git a/include/metall/kernel/segment_allocator.hpp b/include/metall/kernel/segment_allocator.hpp index cd33faba..a21e6c47 100644 --- a/include/metall/kernel/segment_allocator.hpp +++ b/include/metall/kernel/segment_allocator.hpp @@ -187,11 +187,17 @@ class segment_allocator { /// \brief Checks if all memory is deallocated. bool all_memory_deallocated() const { + if (m_chunk_directory.size() == 0) { + return true; + } + #ifndef METALL_DISABLE_OBJECT_CACHE - if (!priv_check_all_small_allocations_are_deallocated()) - return false; + if (priv_check_all_small_allocations_are_cached() && m_chunk_directory.num_used_large_chunks() == 0) { + return true; + } #endif - return m_chunk_directory.get_all_large_chunks().empty(); + + return false; } /// \brief @@ -536,7 +542,8 @@ class segment_allocator { #endif #ifndef METALL_DISABLE_OBJECT_CACHE - bool priv_check_all_small_allocations_are_deallocated() const { + /// \brief Checks if all marked (used) slots in the chunk directory exist in the object cache. + bool priv_check_all_small_allocations_are_cached() const { const auto marked_slots = m_chunk_directory.get_all_marked_slots(); std::set small_allocs; for (const auto &item : marked_slots) { From 3165daa81d2f1c2d3e13938ecd8353695eb3a0c1 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Tue, 4 Oct 2022 16:43:45 -0700 Subject: [PATCH 14/17] CMake CMP0135 is NEW. --- CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b10cee8c..6e001bf4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -5,7 +5,13 @@ include(FetchContent) # -------------------------------------------------------------------------------- # # CMake policy # -------------------------------------------------------------------------------- # -cmake_policy(SET CMP0077 NEW) +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.13") + cmake_policy(SET CMP0077 NEW) +endif() + +if(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.24") + cmake_policy(SET CMP0135 NEW) +endif() # -------------------------------------------------------------------------------- # # Metall general configuration @@ -85,7 +91,6 @@ option(BUILD_TEST "Build the test" OFF) option(RUN_LARGE_SCALE_TEST "Run large scale tests" OFF) option(RUN_BUILD_AND_TEST_WITH_CI "Perform build and basic test with CI" OFF) option(BUILD_VERIFICATION "Build verification directory" OFF) -option(LOGGING "Logging" OFF) option(USE_SORTED_BIN "Use VM space aware algorithm in the bin directory" OFF) set(DEFAULT_VM_RESERVE_SIZE "0" CACHE STRING @@ -192,11 +197,6 @@ if (INITIAL_SEGMENT_SIZE GREATER 0) message(STATUS "METALL_INITIAL_SEGMENT_SIZE=${INITIAL_SEGMENT_SIZE}") endif () -if (LOGGING) - list(APPEND METALL_DEFS "METALL_ENABLE_LOGGING") - message(STATUS "Enable logging") -endif () - if (USE_SORTED_BIN) list(APPEND METALL_DEFS "METALL_USE_SORTED_BIN") message(STATUS "Use VM space aware algorithm in the bin directory") From 6c76a155f8e7f143561b65f851413504cf97b41f Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Tue, 4 Oct 2022 16:45:05 -0700 Subject: [PATCH 15/17] Brush up test scripts. --- scripts/CI/build_and_test.sh | 81 +++--------------- scripts/release_test/full_build_and_test.sh | 95 ++++----------------- scripts/test_kernel.sh | 74 ++++++++++++++++ scripts/test_utility.sh | 11 +++ 4 files changed, 117 insertions(+), 144 deletions(-) create mode 100644 scripts/test_kernel.sh diff --git a/scripts/CI/build_and_test.sh b/scripts/CI/build_and_test.sh index 679edeab..af843c62 100755 --- a/scripts/CI/build_and_test.sh +++ b/scripts/CI/build_and_test.sh @@ -2,6 +2,7 @@ ############################################################################## # Bash script that builds and tests Metall with many compile time configurations +# # 1. Set environmental variables for build # Set manually: # export CC=gcc @@ -12,6 +13,9 @@ # spack load g++ # spack load boost # +# Metall's CMake configuration step downloads the Boost C++ libraries automatically +# if the library is not found. +# # 2. Set optional environmental variables for test # export METALL_TEST_DIR=/tmp # export METALL_LIMIT_MAKE_PARALLELS=n @@ -21,83 +25,26 @@ # sh ./scripts/CI/build_and_test.sh ############################################################################## -####################################### -# Builds and runs test programs -# Globals: -# METALL_ROOT_DIR -# METALL_TEST_DIR -# METALL_LIMIT_MAKE_PARALLELS (option) -# Arguments: -# CMake options to pass -# Outputs: STDOUT and STDERR -####################################### -run_build_and_test_core() { - local BUILD_DIR=./build - - mkdir -p ${BUILD_DIR} - pushd ${BUILD_DIR} - echo "Build and test in ${PWD}" - - # Build - local CMAKE_OPTIONS="$@" - local CMAKE_FILE_LOCATION=${METALL_ROOT_DIR} - or_die cmake ${CMAKE_FILE_LOCATION} ${CMAKE_OPTIONS} - if [[ -z "${METALL_LIMIT_MAKE_PARALLELS}" ]]; then - or_die make -j - else - or_die make -j${METALL_LIMIT_MAKE_PARALLELS} - fi - - # Test 1 - rm -rf ${METALL_TEST_DIR} - or_die ctest --timeout 1000 - - # Test 2 - rm -rf ${METALL_TEST_DIR} - pushd bench/adjacency_list - or_die bash ${METALL_ROOT_DIR}/bench/adjacency_list/test/test.sh -d${METALL_TEST_DIR} - popd - - # Test 3 - rm -rf ${METALL_TEST_DIR} - pushd bench/adjacency_list - or_die bash ${METALL_ROOT_DIR}/bench/adjacency_list/test/test_large.sh -d${METALL_TEST_DIR} - popd - - # TODO: reflink test and C_API test - - rm -rf ${METALL_TEST_DIR} - - popd - rm -rf ${BUILD_DIR} -} - -####################################### -# Show some system information -# Outputs: STDOUT and STDERR -####################################### -show_system_info() { - exec_cmd df -h - exec_cmd df -ih - exec_cmd free -g - exec_cmd uname -r -} - ####################################### # main function # Globals: -# METALL_BUILD_TYPE (option) +# METALL_BUILD_DIR (option, defined if not given) # METALL_TEST_DIR (option, defined if not given) # METALL_ROOT_DIR (defined in this function, readonly) +# METALL_BUILD_TYPE (option) # Outputs: STDOUT and STDERR ####################################### main() { readonly METALL_ROOT_DIR=${PWD} + source ${METALL_ROOT_DIR}/scripts/test_kernel.sh source ${METALL_ROOT_DIR}/scripts/test_utility.sh + echo "Build and test on ${HOSTNAME}" show_system_info - echo "Build and test on ${HOSTNAME}" + if [[ -z "${METALL_BUILD_DIR}" ]]; then + readonly METALL_BUILD_DIR="${METALL_ROOT_DIR}/build_${RANDOM}" + fi setup_test_dir export METALL_TEST_DIR @@ -111,7 +58,8 @@ main() { for DISABLE_FREE_FILE_SPACE in OFF; do for DISABLE_SMALL_OBJECT_CACHE in OFF; do for FREE_SMALL_OBJECT_SIZE_HINT in 0; do - run_build_and_test_core -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + run_build_and_test_kernel \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DDISABLE_FREE_FILE_SPACE=${DISABLE_FREE_FILE_SPACE} \ -DDISABLE_SMALL_OBJECT_CACHE=${DISABLE_SMALL_OBJECT_CACHE} \ -DFREE_SMALL_OBJECT_SIZE_HINT=${FREE_SMALL_OBJECT_SIZE_HINT} \ @@ -124,8 +72,7 @@ main() { -DBUILD_EXAMPLE=ON \ -DRUN_BUILD_AND_TEST_WITH_CI=ON \ -DBUILD_VERIFICATION=OFF \ - -DVERBOSE_SYSTEM_SUPPORT_WARNING=OFF \ - -DLOGGING=OFF + -DVERBOSE_SYSTEM_SUPPORT_WARNING=OFF done done done diff --git a/scripts/release_test/full_build_and_test.sh b/scripts/release_test/full_build_and_test.sh index 89ef2693..340b2c6e 100644 --- a/scripts/release_test/full_build_and_test.sh +++ b/scripts/release_test/full_build_and_test.sh @@ -4,16 +4,19 @@ # Bash script that builds and tests Metall with all compile time configurations # This test would take a few hours at least # -# 1. Set environmental variables for build +# 1. Set environmental variables for build, if needed # Set manually: # export CC=gcc # export CXX=g++ -# export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:/path/to/boost +# export CMAKE_PREFIX_PATH=/path/to/boost:${CMAKE_PREFIX_PATH} # # Or, configure environmental variables using spack: # spack load g++ # spack load boost # +# Metall's CMake configuration step downloads the Boost C++ libraries automatically +# if the library is not found. +# # 2. Set optional environmental variables for test # export METALL_TEST_DIR=/tmp # export METALL_BUILD_DIR=./build @@ -23,105 +26,43 @@ # sh ./scripts/release_test/full_build_and_test.sh ############################################################################## -####################################### -# Builds documents -# Globals: -# METALL_ROOT_DIR -# METALL_BUILD_DIR -# Outputs: STDOUT and STDERR -####################################### -build_docs() { - mkdir -p ${METALL_BUILD_DIR} - cd ${METALL_BUILD_DIR} - echo "Build and test in ${PWD}" - - # Build - local CMAKE_FILE_LOCATION=${METALL_ROOT_DIR} - or_die cmake ${CMAKE_FILE_LOCATION} -DBUILD_DOC=ON - or_die make build_doc - - cd ../ - rm -rf ${METALL_BUILD_DIR} -} - -####################################### -# Builds and runs test programs -# Globals: -# METALL_ROOT_DIR -# METALL_TEST_DIR -# METALL_BUILD_DIR -# METALL_LIMIT_MAKE_PARALLELS (option) -# Arguments: -# CMake options to pass -# Outputs: STDOUT and STDERR -####################################### -run_build_and_test_core() { - mkdir -p ${METALL_BUILD_DIR} - pushd ${METALL_BUILD_DIR} - echo "Build and test in ${PWD}" - - # Build - local CMAKE_OPTIONS="$@" - local CMAKE_FILE_LOCATION=${METALL_ROOT_DIR} - or_die cmake ${CMAKE_FILE_LOCATION} ${CMAKE_OPTIONS} - if [[ -z "${METALL_LIMIT_MAKE_PARALLELS}" ]]; then - or_die make -j - else - or_die make -j${METALL_LIMIT_MAKE_PARALLELS} - fi - - # Test 1 - rm -rf ${METALL_TEST_DIR} - or_die ctest --timeout 1000 - - # Test 2 - rm -rf ${METALL_TEST_DIR} - pushd bench/adjacency_list - or_die bash ./test/test.sh -d${METALL_TEST_DIR} - popd - - # Test 3 - rm -rf ${METALL_TEST_DIR} - pushd bench/adjacency_list - or_die bash ./test/test_large.sh -d${METALL_TEST_DIR} - popd - - # TODO: reflink test and C_API test - - rm -rf ${METALL_TEST_DIR} - popd - rm -rf ${METALL_BUILD_DIR} -} - ####################################### # main function # Globals: -# METALL_TEST_DIR (option, defined if not given) # METALL_BUILD_DIR (option, defined if not given) +# METALL_TEST_DIR (option, defined if not given) # METALL_ROOT_DIR (defined in this function, readonly) # Outputs: STDOUT and STDERR ####################################### main() { readonly METALL_ROOT_DIR=${PWD} + source ${METALL_ROOT_DIR}/scripts/test_kernel.sh source ${METALL_ROOT_DIR}/scripts/test_utility.sh + echo "Build and test on ${HOSTNAME}" + show_system_info + if [[ -z "${METALL_BUILD_DIR}" ]]; then readonly METALL_BUILD_DIR="${METALL_ROOT_DIR}/build_${RANDOM}" fi - # Build documents only + setup_test_dir + export METALL_TEST_DIR + + # Build documents build_docs for BUILD_TYPE in Debug RelWithDebInfo Release; do for DISABLE_FREE_FILE_SPACE in ON OFF; do for DISABLE_SMALL_OBJECT_CACHE in ON OFF; do for FREE_SMALL_OBJECT_SIZE_HINT in 0 8 4096 65536; do - for LOGGING in ON OFF; do - run_build_and_test_core -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ + for USE_ANONYMOUS_NEW_MAP in ON OFF; do + run_build_and_test_kernel \ + -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DDISABLE_FREE_FILE_SPACE=${DISABLE_FREE_FILE_SPACE} \ -DDISABLE_SMALL_OBJECT_CACHE=${DISABLE_SMALL_OBJECT_CACHE} \ -DFREE_SMALL_OBJECT_SIZE_HINT=${FREE_SMALL_OBJECT_SIZE_HINT} \ - -DLOGGING=${LOGGING} \ + -DUSE_ANONYMOUS_NEW_MAP=${USE_ANONYMOUS_NEW_MAP} \ -DBUILD_BENCH=ON \ -DBUILD_TEST=ON \ -DRUN_LARGE_SCALE_TEST=ON \ diff --git a/scripts/test_kernel.sh b/scripts/test_kernel.sh new file mode 100644 index 00000000..f8c2848a --- /dev/null +++ b/scripts/test_kernel.sh @@ -0,0 +1,74 @@ +####################################### +# Builds and runs test programs +# Globals: +# METALL_ROOT_DIR +# METALL_TEST_DIR +# METALL_BUILD_DIR +# METALL_LIMIT_MAKE_PARALLELS (option) +# Arguments: +# CMake options to pass +# Outputs: STDOUT and STDERR +####################################### +run_build_and_test_kernel() { + source ${METALL_ROOT_DIR}/scripts/test_utility.sh + + mkdir -p ${METALL_BUILD_DIR} + pushd ${METALL_BUILD_DIR} + echo "Build and test in ${PWD}" + + # Build + local CMAKE_OPTIONS="$@" + local CMAKE_FILE_LOCATION=${METALL_ROOT_DIR} + or_die cmake ${CMAKE_FILE_LOCATION} ${CMAKE_OPTIONS} + if [[ -z "${METALL_LIMIT_MAKE_PARALLELS}" ]]; then + or_die make -j + else + or_die make -j${METALL_LIMIT_MAKE_PARALLELS} + fi + + # Test 1 + rm -rf ${METALL_TEST_DIR} + or_die ctest --timeout 1000 + + # Test 2 + rm -rf ${METALL_TEST_DIR} + pushd bench/adjacency_list + or_die bash ./test/test.sh -d${METALL_TEST_DIR} + popd + + # Test 3 + rm -rf ${METALL_TEST_DIR} + pushd bench/adjacency_list + or_die bash ./test/test_large.sh -d${METALL_TEST_DIR} + popd + + # TODO: reflink test and C_API test + + rm -rf ${METALL_TEST_DIR} + popd + rm -rf ${METALL_BUILD_DIR} +} + + +####################################### +# Builds documents +# Globals: +# METALL_ROOT_DIR +# METALL_BUILD_DIR +# Outputs: STDOUT and STDERR +####################################### +build_docs() { + source ${METALL_ROOT_DIR}/scripts/test_utility.sh + + mkdir -p ${METALL_BUILD_DIR} + cd ${METALL_BUILD_DIR} + echo "Build and test in ${PWD}" + + # Build + local CMAKE_FILE_LOCATION=${METALL_ROOT_DIR} + or_die cmake ${CMAKE_FILE_LOCATION} -DBUILD_DOC=ON + or_die make build_doc + + cd ../ + rm -rf ${METALL_BUILD_DIR} +} \ No newline at end of file diff --git a/scripts/test_utility.sh b/scripts/test_utility.sh index 01b46179..d313b8f6 100644 --- a/scripts/test_utility.sh +++ b/scripts/test_utility.sh @@ -54,4 +54,15 @@ setup_test_dir() { # mkdir -p ${METALL_TEST_DIR} # Metall creates automatically if the directory does not exist echo "Store test data to ${METALL_TEST_DIR}" +} + +####################################### +# Show some system information +# Outputs: STDOUT and STDERR +####################################### +show_system_info() { + exec_cmd df -h + exec_cmd df -ih + exec_cmd free -g + exec_cmd uname -r } \ No newline at end of file From ca1db27a9f667433d034294dbc99082ede9932eb Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Wed, 5 Oct 2022 05:48:34 -0700 Subject: [PATCH 16/17] Minor bugfix in test_kernel.sh --- scripts/test_kernel.sh | 39 +++++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/scripts/test_kernel.sh b/scripts/test_kernel.sh index f8c2848a..b4f75acb 100644 --- a/scripts/test_kernel.sh +++ b/scripts/test_kernel.sh @@ -12,8 +12,18 @@ run_build_and_test_kernel() { source ${METALL_ROOT_DIR}/scripts/test_utility.sh - mkdir -p ${METALL_BUILD_DIR} - pushd ${METALL_BUILD_DIR} + local test_dir=${METALL_TEST_DIR} + if [ -d ${test_dir} ]; then + test_dir="${test_dir}/metall-test-${RANDOM}" + fi + + local build_dir=${METALL_BUILD_DIR} + if [ -d ${build_dir} ]; then + build_dir="${build_dir}/metall-build-${RANDOM}" + fi + + mkdir -p ${build_dir} + pushd ${build_dir} echo "Build and test in ${PWD}" # Build @@ -27,26 +37,26 @@ run_build_and_test_kernel() { fi # Test 1 - rm -rf ${METALL_TEST_DIR} + rm -rf ${test_dir} or_die ctest --timeout 1000 # Test 2 - rm -rf ${METALL_TEST_DIR} + rm -rf ${test_dir} pushd bench/adjacency_list - or_die bash ./test/test.sh -d${METALL_TEST_DIR} + or_die bash ./test/test.sh -d${test_dir} popd # Test 3 - rm -rf ${METALL_TEST_DIR} + rm -rf ${test_dir} pushd bench/adjacency_list - or_die bash ./test/test_large.sh -d${METALL_TEST_DIR} + or_die bash ./test/test_large.sh -d${test_dir} popd # TODO: reflink test and C_API test - rm -rf ${METALL_TEST_DIR} + rm -rf ${test_dir} popd - rm -rf ${METALL_BUILD_DIR} + rm -rf ${build_dir} } @@ -60,8 +70,13 @@ run_build_and_test_kernel() { build_docs() { source ${METALL_ROOT_DIR}/scripts/test_utility.sh - mkdir -p ${METALL_BUILD_DIR} - cd ${METALL_BUILD_DIR} + local build_dir=${METALL_BUILD_DIR} + if [ -d ${build_dir} ]; then + build_dir="${build_dir}/metall-build-${RANDOM}" + fi + + mkdir -p ${build_dir} + cd ${build_dir} echo "Build and test in ${PWD}" # Build @@ -70,5 +85,5 @@ build_docs() { or_die make build_doc cd ../ - rm -rf ${METALL_BUILD_DIR} + rm -rf ${build_dir} } \ No newline at end of file From 854bd331c3528d9df200667eec3d83c7b3da9ed2 Mon Sep 17 00:00:00 2001 From: Keita Iwabuchi Date: Tue, 11 Oct 2022 18:38:26 -0700 Subject: [PATCH 17/17] Release v0.22 --- CMakeLists.txt | 2 +- docs/Doxyfile.in | 2 +- include/metall/version.hpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6e001bf4..d18bed89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ endif() # Metall general configuration # -------------------------------------------------------------------------------- # project(Metall - VERSION 0.21 + VERSION 0.22 DESCRIPTION "A persistent memory allocator for data-centric analytics" HOMEPAGE_URL "https://github.com/LLNL/metall") diff --git a/docs/Doxyfile.in b/docs/Doxyfile.in index 389a693e..6bb3c03f 100644 --- a/docs/Doxyfile.in +++ b/docs/Doxyfile.in @@ -38,7 +38,7 @@ PROJECT_NAME = "Metall" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = v0.21 +PROJECT_NUMBER = v0.22 # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/include/metall/version.hpp b/include/metall/version.hpp index d02c9277..9e3f314d 100644 --- a/include/metall/version.hpp +++ b/include/metall/version.hpp @@ -14,7 +14,7 @@ /// METALL_VERSION / 100 % 1000 // the minor version. /// METALL_VERSION % 100 // the patch level. /// \endcode -#define METALL_VERSION 2100 +#define METALL_VERSION 2200 namespace metall { /// \brief Variable type to handle a version data.