diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index c06500b..83643a6 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -22,6 +22,7 @@ build_example(event_trace) build_example(exception) build_example(logging) build_example(mapped_file) +build_example(mapped_vector) build_example(nlp) build_example(parse1) build_example(parse2) diff --git a/examples/example_btree.cpp b/examples/example_btree.cpp index d9daafc..1b57334 100644 --- a/examples/example_btree.cpp +++ b/examples/example_btree.cpp @@ -3,11 +3,12 @@ * @copyright David Mott (c) 2016. Distributed under the Boost Software License Version 1.0. See LICENSE.md or http://boost.org/LICENSE_1_0.txt for details. */ -#if 1 +#if 0 int main(){} #else #include #include +#include int main(){ auto oFile = xtd::filesystem::temp_directory_path() + xtd::executable::this_executable().path().filename(); @@ -15,10 +16,16 @@ int main(){ try{ xtd::btree oTree(oFile); for (int i = 0; i < 10000 && oTree.insert(i, i); i++); + return 0; + } + catch(const xtd::exception& ex){ + std::cout << "An xtd::exception occurred at " << ex.location().file() << "(" << ex.location().line() << ") : " << ex.what() << std::endl; + } + catch(const std::exception& ex){ + std::cout << "An std::exception occurred: " << ex.what() << std::endl; } catch (...){} - - xtd::filesystem::remove(oFile); - return 0; + return -1; +// xtd::filesystem::remove(oFile); } #endif \ No newline at end of file diff --git a/examples/example_mapped_file.cpp b/examples/example_mapped_file.cpp index b282a21..2e9d0aa 100644 --- a/examples/example_mapped_file.cpp +++ b/examples/example_mapped_file.cpp @@ -20,7 +20,7 @@ int main(){ xtd::filesystem::path oFile = xtd::filesystem::temp_directory_path() + "tmp_mapped_file.dat"; { - xtd::mapped_file oMappedFile(oFile); + xtd::mapped_file<-1> oMappedFile(oFile); auto oPerson = oMappedFile.get(0); oPerson->age = 123; oPerson->ssn = 456; @@ -28,7 +28,7 @@ int main(){ strcpy(oPerson->last_name, "Marx"); } { - xtd::mapped_file oMappedFile(oFile); + xtd::mapped_file<-1> oMappedFile(oFile); auto oPerson = oMappedFile.get(1); oPerson->age = 456; oPerson->ssn = 789; @@ -36,7 +36,7 @@ int main(){ strcpy(oPerson->last_name, "Marx"); } - xtd::mapped_file oMappedFile(oFile); + xtd::mapped_file<-1> oMappedFile(oFile); auto oGroucho = oMappedFile.get(0); DBG(oGroucho->first_name, " ", oGroucho->last_name); auto oHarpo = oMappedFile.get(1); diff --git a/examples/example_mapped_vector.cpp b/examples/example_mapped_vector.cpp new file mode 100644 index 0000000..8857c98 --- /dev/null +++ b/examples/example_mapped_vector.cpp @@ -0,0 +1,36 @@ +/** @file +* demonstrates vector on a memory mapped file +* @copyright David Mott (c) 2016. Distributed under the Boost Software License Version 1.0. See LICENSE.md or http://boost.org/LICENSE_1_0.txt for details. +*/ + +#include + +struct person{ + int age; + char first_name[50]; + char last_name[50]; + int ssn; + +}; + +int main(){ + try{ + xtd::filesystem::path oFile = xtd::filesystem::temp_directory_path() + "tmp_mapped_vector.dat"; + + xtd::mapped_vector oInts(oFile); + + for (int i=0 ; i<10000000 ; i++){ + oInts.push_back(i); + } + + return 0; + } + catch (const xtd::crt_exception& ex){ + ERR("A CRT exception ", ex.errnum(), " occurred: ", ex.what()); + return -1; + } + catch (const xtd::exception& ex){ + ERR("An exception occurred: ", ex.what()); + return -1; + } +} diff --git a/include/xtd/btree.hpp b/include/xtd/btree.hpp index 4ff4f14..3b1ea19 100644 --- a/include/xtd/btree.hpp +++ b/include/xtd/btree.hpp @@ -61,29 +61,29 @@ namespace xtd{ template typename _Ty::pointer get(size_t Page){ - for (_super_t::iterator oItem = _super_t::begin(); _super_t::end() != oItem; ++oItem){ - if (std::get<0>(*oItem) == Page){ + for (typename _super_t::iterator oItem = _super_t::begin(); _super_t::end() != oItem; ++oItem){ + if (oItem->first == Page){ if (oItem == _super_t::begin()){ - return oItem->second.cast<_Ty>(); + return oItem->second.template cast<_Ty>(); } - data_page<_PageSize>::pointer oRet = std::get<1>(*oItem); + auto oRet = oItem->second; _super_t::erase(oItem); _super_t::push_front(std::make_pair(Page, oRet)); - return oRet.cast<_Ty>(); + return oRet.template cast<_Ty>(); } } - if (size() >= _CacheSize) _super_t::pop_back(); + if (_super_t::size() >= _CacheSize) _super_t::pop_back(); _super_t::push_front(std::make_pair(Page, _File.get>(Page))); - return _super_t::front().second.cast<_Ty>(); + return _super_t::front().second.template cast<_Ty>(); } template typename _Ty::pointer append(size_t& newpage){ - if (size() >= _CacheSize) _super_t::pop_back(); - auto pNewPage = _File.append>(newpage); + if (_super_t::size() >= _CacheSize) _super_t::pop_back(); + auto pNewPage = _File.template append>(newpage); _super_t::push_front(std::make_pair(newpage, pNewPage)); pNewPage->_page_type = _Ty::type; - return pNewPage.cast<_Ty>(); + return pNewPage.template cast<_Ty>(); } }; @@ -141,7 +141,7 @@ namespace xtd{ template void split(_CacheT& oCache, size_t left_page_idx, key_type& left_page_key, size_t& right_page_idx){ - auto oNewPage = oCache.append(right_page_idx); + auto oNewPage = oCache.template append(right_page_idx); D_(const leaf * pNewPage = oNewPage.get()); left_page_key = _records[records_per_page() / 2]._key; for (size_t i = 1+(records_per_page() / 2); i < records_per_page(); ++i, ++oNewPage->_page_header._count, --_page_header._count){ @@ -158,8 +158,7 @@ namespace xtd{ template class branch : public data_page<_PageSize>{ using _super_t = data_page<_PageSize>; - using leaf = leaf<_KeyT, _ValueT, _PageSize>; - using data_page = data_page<_PageSize>; + using leaf_t = leaf<_KeyT, _ValueT, _PageSize>; public: using pointer = typename xtd::mapped_file<_PageSize>::template mapped_page>; using key_type = _KeyT; @@ -189,15 +188,15 @@ namespace xtd{ template inline bool insert(_CacheT& oCache, const key_type& key, const value_type& value){ - typename data_page::pointer oPage; + typename _super_t::pointer oPage; for(;;){ for (size_t i = 0; i < _page_header._count; ++i){ if (key <= _records[i]._key){ - oPage = oCache.get(_records[i]._left); + oPage = oCache.template get<_super_t>(_records[i]._left); if (page_type::leaf_page == oPage->_page_type){ - auto oLeaf = oPage.cast(); - D_(const leaf * pLeaf = oLeaf.get()); - if (oLeaf->_page_header._count >= leaf::records_per_page()){ + auto oLeaf = oPage.template cast(); + D_(const leaf_t * pLeaf = oLeaf.get()); + if (oLeaf->_page_header._count >= leaf_t::records_per_page()){ _records[_page_header._count]._left = _records[i]._left; oLeaf->split(oCache, _records[i]._left, _records[_page_header._count]._key, _records[i]._left); std::sort(&_records[0], &_records[_page_header._count], [](const record& lhs, const record& rhs){ return lhs._key < rhs._key; }); @@ -206,7 +205,7 @@ namespace xtd{ } return oLeaf->insert(key, value); } else{ - auto oBranch = oPage.cast(); + auto oBranch = oPage.template cast(); D_(const branch * pBranch = oBranch.get()); if (oBranch->_page_header._count >= branch::records_per_page()){ _records[_page_header._count] = _records[i]; @@ -219,11 +218,11 @@ namespace xtd{ } } } - oPage = oCache.get(_page_header._right); + oPage = oCache.template get<_super_t>(_page_header._right); if (page_type::leaf_page == oPage->_page_type){ - auto oLeaf = oPage.cast(); - D_(const leaf * pLeaf = oLeaf.get()); - if (oLeaf->_page_header._count >= leaf::records_per_page()){ + auto oLeaf = oPage.template cast(); + D_(const leaf_t * pLeaf = oLeaf.get()); + if (oLeaf->_page_header._count >= leaf_t::records_per_page()){ _records[_page_header._count]._left = _page_header._right; oLeaf->split(oCache, _page_header._right, _records[_page_header._count]._key, _page_header._right); std::sort(&_records[0], &_records[_page_header._count], [](const record& lhs, const record& rhs){ return lhs._key < rhs._key; }); @@ -232,7 +231,7 @@ namespace xtd{ } return oLeaf->insert(key, value); } else { - auto oBranch = oPage.cast(); + auto oBranch = oPage.template cast(); D_(const branch * pBranch = oBranch.get()); if (oBranch->_page_header._count < branch::records_per_page()){ _records[_page_header._count]._left = _page_header._right; @@ -248,7 +247,7 @@ namespace xtd{ } template inline void split(_CacheT& oCache, key_type& left_page_key, size_t & right_page_idx){ - auto oNewPage = oCache.append(right_page_idx); + auto oNewPage = oCache.template append(right_page_idx); D_(const branch * pNewPage = oNewPage.get()); TODO("convert to memcpy") for (size_t i = 1+records_per_page() / 2; i < records_per_page(); ++i, ++oNewPage->_page_header._count, --_page_header._count){ @@ -275,15 +274,15 @@ namespace xtd{ private: _::btree::lru_cache<_PageSize, _CacheSize> _cache; using file_header = _::btree::file_header<_KeyT, _ValueT, _PageSize>; - using data_page = _::btree::data_page<_PageSize>; - using leaf = typename _::btree::leaf<_KeyT, _ValueT, _PageSize>; - using branch = _::btree::branch<_KeyT, _ValueT, _PageSize>; + using data_page_t = _::btree::data_page<_PageSize>; + using leaf_t = _::btree::leaf<_KeyT, _ValueT, _PageSize>; + using branch_t = _::btree::branch<_KeyT, _ValueT, _PageSize>; using page_type = _::btree::page_type; typename file_header::pointer _file_header; public: - btree(const xtd::filesystem::path& oPath) : _cache(oPath), _file_header(_cache.get(0)){} + btree(const xtd::filesystem::path& oPath) : _cache(oPath), _file_header(_cache.template get(0)){} /** insert a value in the container * @@ -294,8 +293,8 @@ namespace xtd{ bool insert(const key_type& key, const value_type& value){ //no root page if (0 == _file_header->_root_page){ - auto oLeaf = _cache.append(_file_header->_root_page); - D_(const leaf * pLeaf = oLeaf.get()); + auto oLeaf = _cache.template append(_file_header->_root_page); + D_(const leaf_t * pLeaf = oLeaf.get()); if (oLeaf->insert(key, value)){ _file_header->_count++; return true; @@ -303,20 +302,20 @@ namespace xtd{ return false; } - auto oRoot = _cache.get(_file_header->_root_page); + auto oRoot = _cache.template get(_file_header->_root_page); if (page_type::branch_page == oRoot->_page_type){ //root is a branch page //root page is full - auto oBranch = oRoot.cast(); - D_(const branch * pBranch = oBranch.get()); - if (oBranch->_page_header._count >= branch::records_per_page()){ + auto oBranch = oRoot.template cast(); + D_(const branch_t * pBranch = oBranch.get()); + if (oBranch->_page_header._count >= branch_t::records_per_page()){ size_t iNewRoot; - auto oNewRoot = _cache.append(iNewRoot); + auto oNewRoot = _cache.template append(iNewRoot); oBranch->split(_cache, oNewRoot->_records[0]._key, oNewRoot->_page_header._right); oNewRoot->_records[0]._left = _file_header->_root_page; oNewRoot->_page_header._count = 1; _file_header->_root_page = iNewRoot; - oBranch = oNewRoot; + oBranch = std::move(oNewRoot); } // root page is not full if (oBranch->insert(_cache, key, value)){ @@ -326,10 +325,10 @@ namespace xtd{ return false; } // root is a leaf page - auto oLeaf = oRoot.cast(); - D_(const leaf * pLeaf = oLeaf.get()); + auto oLeaf = oRoot.template cast(); + D_(const leaf_t * pLeaf = oLeaf.get()); //leaf is not full - if (oLeaf->_page_header._count < leaf::records_per_page()){ + if (oLeaf->_page_header._count < leaf_t::records_per_page()){ if (oLeaf->insert(key, value)){ _file_header->_count++; return true; @@ -338,7 +337,7 @@ namespace xtd{ } //leaf is full auto iOldRootIdx = _file_header->_root_page; - auto oNewRoot = _cache.append(_file_header->_root_page); + auto oNewRoot = _cache.template append(_file_header->_root_page); oNewRoot->_records[0]._left = iOldRootIdx; oLeaf->split(_cache, iOldRootIdx, oNewRoot->_records[0]._key, oNewRoot->_page_header._right); oNewRoot->_page_header._count = 1; diff --git a/include/xtd/mapped_file.hpp b/include/xtd/mapped_file.hpp index f02b14d..a130b51 100644 --- a/include/xtd/mapped_file.hpp +++ b/include/xtd/mapped_file.hpp @@ -36,14 +36,14 @@ namespace xtd{ return iRet; } }; - } #if ((XTD_OS_CYGWIN|XTD_OS_MSYS|XTD_OS_LINUX) & XTD_OS) - class mapped_file{ + template + class mapped_file : _::mapped_file_base<_page_size>{ + using _super_t = _::mapped_file_base<_page_size>; int _FileNum; - size_t _PageSize; public: ~mapped_file(){ close(_FileNum); } @@ -53,47 +53,31 @@ namespace xtd{ {} + template class mapped_page : public std::shared_ptr <_Ty>{ - template class mapped_page{ + using _super_t = std::shared_ptr<_Ty>; + template friend class mapped_file; + explicit mapped_page(void * addr) : _super_t(reinterpret_cast<_Ty*>(addr), [](_Ty*addr){ munmap(addr, _::mapped_file_base<_page_size>::page_size());}){} + template mapped_page(_ArgTs...oArgs) : _super_t(std::forward<_ArgTs>(oArgs)...){} public: - using value_type = _Ty; - - ~mapped_page(){ munmap(_Page, memory::page_size()); } - mapped_page(const mapped_page&) = delete; - mapped_page& operator=(const mapped_page&) = delete; + mapped_page() : _super_t(nullptr, [](_Ty*addr){ munmap(addr, _::mapped_file_base<_page_size>::page_size());}){} - mapped_page(mapped_page&& src) : _Page(nullptr){ - std::swap(_Page, src._Page); - } - mapped_page& operator=(mapped_page&& src){ - if (this != &src) { - std::swap(_Page, src._Page); - } - return *this; + void flush(){ + xtd::crt_exception::throw_if(msync(_super_t::get(), _::mapped_file_base<_page_size>::page_size(), MS_SYNC), [](int i){return 0 != i;}); } - value_type* get() { return _Page; } - const value_type* get() const { return _Page; } - - value_type* operator->() { return get();} - const value_type* operator->() const { return get();} - - value_type& operator*() { XTD_ASSERT(get()); return *get();} - const value_type& operator*() const { XTD_ASSERT(get()); return *get();} - - void flush(){ - xtd::crt_exception::throw_if(msync(_Page, memory::page_size(), MS_SYNC), [](int i){return 0 != i;}); + template + mapped_page<_OtherT> cast(){ + return (mapped_page<_OtherT>(*this, static_cast<_OtherT*>(get()))); } - protected: - friend class mapped_file; - explicit mapped_page(void * addr) : _Page(reinterpret_cast(addr)){} - value_type* _Page; + }; + template mapped_page<_Ty> get(size_t pageNum){ - auto iPageSize = memory::page_size(); + auto iPageSize = _super_t::page_size(); struct stat oStat; xtd::crt_exception::throw_if(fstat(_FileNum, &oStat), [](int i){ return -1 == i; }); off_t iLastByte = (pageNum * iPageSize) + iPageSize; @@ -107,6 +91,21 @@ namespace xtd{ [](void*addr){ return nullptr==addr || MAP_FAILED==addr; } )); } + + template mapped_page<_Ty> append(size_t& newpage){ + auto iPageSize = _super_t::page_size(); + struct stat oStat; + xtd::crt_exception::throw_if(fstat(_FileNum, &oStat), [](int i){ return -1 == i; }); + newpage = oStat.st_size / iPageSize; + xtd::crt_exception::throw_if(lseek(_FileNum, oStat.st_size + iPageSize, SEEK_SET), [](long l){ return -1==l;}); + xtd::crt_exception::throw_if(write(_FileNum, "", 1), [](int i){ return 1 != i;}); + + return mapped_page<_Ty>( + xtd::crt_exception::throw_if( + mmap(nullptr, iPageSize, PROT_READ|PROT_WRITE, MAP_SHARED, _FileNum, oStat.st_size), + [](void*addr){ return nullptr==addr || MAP_FAILED==addr; } + )); + } }; #elif ((XTD_OS_MINGW|XTD_OS_WINDOWS) & XTD_OS) diff --git a/include/xtd/mapped_vector.hpp b/include/xtd/mapped_vector.hpp new file mode 100644 index 0000000..9ca9dd5 --- /dev/null +++ b/include/xtd/mapped_vector.hpp @@ -0,0 +1,59 @@ +/** @file +memory mapped vector +@copyright David Mott (c) 2016. Distributed under the Boost Software License Version 1.0. See LICENSE.md or http://boost.org/LICENSE_1_0.txt for details. +*/ + +#pragma once + +#include + +namespace xtd{ + + template + class mapped_vector : mapped_file<_page_size>{ + friend class iterator; + using _super_t = mapped_file<_page_size>; + + struct file_header_page{ + using pointer = typename _super_t::template mapped_page; + size_t _count; + }; + + struct data_page{ + using pointer = typename _super_t::template mapped_page; + _Ty _values[1]; + static size_t items_per_page(){ return _::mapped_file_base<_page_size>::page_size() / sizeof(_Ty); } + }; + + typename file_header_page::pointer _root; + + public: + using value_type = _Ty; + template mapped_vector(_ArgTs...oArgs) : _super_t(std::forward<_ArgTs>(oArgs)...), _root(_super_t::template get(0)){} + + class iterator{ + template friend class mapped_vector; + using data_page_t = typename mapped_vector<_Ty, _page_size>::data_page; + mapped_vector& _file; + size_t _current_index; + typename _super_t::template mapped_page _current_page; + iterator(size_t index, mapped_vector& oFile) : _current_index(index), _file(oFile){ + if (-1 != _current_index){ + _current_page = _file.get(1 + _current_index / _super_t::page_size()); + } + } + public: + + }; + + void push_back(value_type& value){ + auto oPage = _super_t::template get(1 + _root->_count / data_page::items_per_page()); + oPage->_values[_root->_count % data_page::items_per_page()] = value; + _root->_count++; + } + + iterator end() { return iterator(-1, *this);} + iterator begin(){ return iterator( (_root->_count ? 0 : -1), *this); } + + }; +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 800f285..5961626 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,6 +17,7 @@ set(TEST_SOURCE tests.cpp test_hash_map.hpp test_logging.hpp test_mapped_file.hpp + test_mapped_vector.hpp test_meta.hpp test_parse.hpp test_path.hpp @@ -49,6 +50,7 @@ build_option(TEST_EXCEPTION "test xtd::exception") build_option(TEST_EXCUTABLE "test xtd::executable") build_option(TEST_LOGGING "test xtd::log") build_option(TEST_MAPPED_FILE "test xtd::mapped_file") +build_option(TEST_MAPPED_VECTOR "test xtd::mapped_vector") build_option(TEST_META "test meta programming") build_option(TEST_PARSE "test xtd::parse") build_option(TEST_PATH "test xtd::filesystem::path") diff --git a/tests/test_mapped_file.hpp b/tests/test_mapped_file.hpp index 14896a5..6db32fd 100644 --- a/tests/test_mapped_file.hpp +++ b/tests/test_mapped_file.hpp @@ -24,13 +24,13 @@ struct mapped_file_test_struct{ TEST_F(test_mapped_file, initialization){ auto oPath = xtd::filesystem::temp_directory_path() + xtd::executable::this_executable().path().filename(); - EXPECT_NO_THROW(xtd::mapped_file oFile(oPath)); + EXPECT_NO_THROW(xtd::mapped_file<-1> oFile(oPath)); xtd::filesystem::remove(oPath); } TEST_F(test_mapped_file, page_initialization){ auto oPath = xtd::filesystem::temp_directory_path() + xtd::executable::this_executable().path().filename(); - xtd::mapped_file oFile(xtd::filesystem::temp_directory_path() + xtd::executable::this_executable().path().filename()); + xtd::mapped_file<-1> oFile(xtd::filesystem::temp_directory_path() + xtd::executable::this_executable().path().filename()); EXPECT_NO_THROW(auto oPage = oFile.get(0)); EXPECT_NO_THROW(auto oPage2 = oFile.get(1)); EXPECT_NO_THROW(auto oPage3 = oFile.get(2)); @@ -39,7 +39,7 @@ TEST_F(test_mapped_file, page_initialization){ TEST_F(test_mapped_file, read){ auto oPath = xtd::filesystem::temp_directory_path() + xtd::executable::this_executable().path().filename(); - xtd::mapped_file oFile(oPath); + xtd::mapped_file<-1> oFile(oPath); auto oPage = oFile.get(0); mapped_file_test_struct s; memcpy(&s, oPage.get(), sizeof(mapped_file_test_struct)); @@ -48,7 +48,7 @@ TEST_F(test_mapped_file, read){ TEST_F(test_mapped_file, write){ auto oPath = xtd::filesystem::temp_directory_path() + xtd::executable::this_executable().path().filename(); - xtd::mapped_file oFile(oPath); + xtd::mapped_file<-1> oFile(oPath); { auto oPage = oFile.get(0); oPage->age=123; diff --git a/tests/test_mapped_vector.hpp b/tests/test_mapped_vector.hpp new file mode 100644 index 0000000..ad2752f --- /dev/null +++ b/tests/test_mapped_vector.hpp @@ -0,0 +1,12 @@ +/** @file +xtd::mapped_vector system and unit tests +@copyright David Mott (c) 2016. Distributed under the Boost Software License Version 1.0. See LICENSE.md or http://boost.org/LICENSE_1_0.txt for details. +*/ + +#pragma once + +#include + +TEST(test_mapped_vector, initialization){ + +} \ No newline at end of file diff --git a/tests/tests.cpp b/tests/tests.cpp index 2a6618a..0627a1c 100644 --- a/tests/tests.cpp +++ b/tests/tests.cpp @@ -52,12 +52,16 @@ main system and unit test entry point #include "test_logging.hpp" #endif -#if (ON==TEST_LOGGING) +#if (ON==TEST_META) + #include "test_meta.hpp" +#endif + +#if (ON==TEST_MAPPED_FILE) #include "test_mapped_file.hpp" #endif -#if (ON==TEST_META) - #include "test_meta.hpp" +#if (ON==TEST_MAPPED_VECTOR) + #include "test_mapped_vector.hpp" #endif #if (ON==TEST_PARSE)