From ac62e7a6dd6ddc3139f3438505e09cb3f3ac0ff8 Mon Sep 17 00:00:00 2001 From: Jim Hester Date: Thu, 18 Apr 2019 09:39:51 -0400 Subject: [PATCH] Squashed 'src/mio/' changes from c7c7163a..88e0263b 88e0263b Fix memory mapped data offset 514837b2 Update test.cpp 18fbf199 Fix minor errors in internal docs cafa3136 Merge pull request #40 from patrick--/single_header 3c655b70 Added single header file information to the README 647e9779 Initial single header support using amalgamate 3c658bf6 Merge pull request #38 from amallia/patch-1 d25c02f0 Update mmap.ipp d35a6db8 Update mmap.ipp git-subtree-dir: src/mio git-subtree-split: 88e0263b075b832e78b9a36df58575a96ffe5e7d --- README.md | 7 + include/mio/detail/mmap.ipp | 11 +- include/mio/mmap.hpp | 16 +- single_include/mio/mio.hpp | 1751 +++++++++++++++++++++++++++++++++++ test/test.cpp | 42 +- third_party/LICENSE.md | 28 + third_party/amalgamate.py | 300 ++++++ third_party/config.json | 11 + 8 files changed, 2140 insertions(+), 26 deletions(-) create mode 100644 single_include/mio/mio.hpp create mode 100644 third_party/LICENSE.md create mode 100644 third_party/amalgamate.py create mode 100644 third_party/config.json diff --git a/README.md b/README.md index d915feec..e50b23ff 100644 --- a/README.md +++ b/README.md @@ -181,6 +181,12 @@ using mmap_sink = mio::basic_mmap_sink; Though generally not needed, since mio maps users requested offsets to page boundaries, you can query the underlying system's page allocation granularity by invoking `mio::page_size()`, which is located in `mio/page.hpp`. +### Single Header File +Mio can be added to your project as a single header file simply by including `\single_include\mio\mio.hpp`. Single header files can be regenerated at any time by running the `amalgamate.py` script within `\third_party`. +``` +python amalgamate.py -c config.json -s ../include +``` + ## CMake As a header-only library, mio has no compiled components. Nevertheless, a [CMake](https://cmake.org/overview/) build system is provided to allow easy testing, installation, and subproject composition on many platforms and operating systems. @@ -328,3 +334,4 @@ target_link_libraries( MyTarget PUBLIC ) ``` Note that, as a subproject, mio's tests and examples will not be built and CPack integration is deferred to the host project. + diff --git a/include/mio/detail/mmap.ipp b/include/mio/detail/mmap.ipp index d38a1b88..2e73958d 100644 --- a/include/mio/detail/mmap.ipp +++ b/include/mio/detail/mmap.ipp @@ -124,7 +124,7 @@ file_handle_type open_file(const String& path, const access_mode mode, return handle; } -inline int64_t query_file_size(file_handle_type handle, std::error_code& error) +inline size_t query_file_size(file_handle_type handle, std::error_code& error) { error.clear(); #ifdef _WIN32 @@ -201,7 +201,7 @@ inline mmap_context memory_map(const file_handle_type file_handle, const int64_t } #endif mmap_context ctx; - ctx.data = mapping_start + offset - aligned_offset; + ctx.data = mapping_start + offset; ctx.length = length; ctx.mapped_length = length_to_map; #ifdef _WIN32 @@ -400,9 +400,10 @@ void basic_mmap::unmap() if(data_) { ::munmap(const_cast(get_mapping_start()), mapped_length_); } #endif - // If file_handle_ was obtained by our opening it (when map is called with a path, - // rather than an existing file handle), we need to close it, otherwise it must not - // be closed as it may still be used outside this instance. + // If `file_handle_` was obtained by our opening it (when map is called with + // a path, rather than an existing file handle), we need to close it, + // otherwise it must not be closed as it may still be used outside this + // instance. if(is_handle_internal_) { #ifdef _WIN32 diff --git a/include/mio/mmap.hpp b/include/mio/mmap.hpp index 9de9d93c..ddb95b2d 100644 --- a/include/mio/mmap.hpp +++ b/include/mio/mmap.hpp @@ -76,15 +76,17 @@ struct basic_mmap // Points to the first requested byte, and not to the actual start of the mapping. pointer data_ = nullptr; - // Length, in bytes, requested by user, which may not be the length of the full - // mapping, and the entire length of the full mapping. + // Length--in bytes--requested by user (which may not be the length of the + // full mapping) and the length of the full mapping. size_type length_ = 0; size_type mapped_length_ = 0; - // Letting user map a file using both an existing file handle and a path introcudes - // On POSIX, we only need a file handle to create a mapping, while on Windows - // systems the file handle is necessary to retrieve a file mapping handle, but any - // subsequent operations on the mapped region must be done through the latter. + // Letting user map a file using both an existing file handle and a path + // introcudes some complexity (see `is_handle_internal_`). + // On POSIX, we only need a file handle to create a mapping, while on + // Windows systems the file handle is necessary to retrieve a file mapping + // handle, but any subsequent operations on the mapped region must be done + // through the latter. handle_type file_handle_ = INVALID_HANDLE_VALUE; #ifdef _WIN32 handle_type file_mapping_handle_ = INVALID_HANDLE_VALUE; @@ -94,7 +96,7 @@ struct basic_mmap // introcudes some complexity in that we must not close the file handle if // user provided it, but we must close it if we obtained it using the // provided path. For this reason, this flag is used to determine when to - // close file_handle_. + // close `file_handle_`. bool is_handle_internal_; public: diff --git a/single_include/mio/mio.hpp b/single_include/mio/mio.hpp new file mode 100644 index 00000000..1d43307d --- /dev/null +++ b/single_include/mio/mio.hpp @@ -0,0 +1,1751 @@ +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_MMAP_HEADER +#define MIO_MMAP_HEADER + +// #include "mio/page.hpp" +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_PAGE_HEADER +#define MIO_PAGE_HEADER + +#ifdef _WIN32 +# include +#else +# include +#endif + +namespace mio { + +/** + * This is used by `basic_mmap` to determine whether to create a read-only or + * a read-write memory mapping. + */ +enum class access_mode +{ + read, + write +}; + +/** + * Determines the operating system's page allocation granularity. + * + * On the first call to this function, it invokes the operating system specific syscall + * to determine the page size, caches the value, and returns it. Any subsequent call to + * this function serves the cached value, so no further syscalls are made. + */ +inline size_t page_size() +{ + static const size_t page_size = [] + { +#ifdef _WIN32 + SYSTEM_INFO SystemInfo; + GetSystemInfo(&SystemInfo); + return SystemInfo.dwAllocationGranularity; +#else + return sysconf(_SC_PAGE_SIZE); +#endif + }(); + return page_size; +} + +/** + * Alligns `offset` to the operating's system page size such that it subtracts the + * difference until the nearest page boundary before `offset`, or does nothing if + * `offset` is already page aligned. + */ +inline size_t make_offset_page_aligned(size_t offset) noexcept +{ + const size_t page_size_ = page_size(); + // Use integer division to round down to the nearest page alignment. + return offset / page_size_ * page_size_; +} + +} // namespace mio + +#endif // MIO_PAGE_HEADER + + +#include +#include +#include +#include + +#ifdef _WIN32 +# ifndef WIN32_LEAN_AND_MEAN +# define WIN32_LEAN_AND_MEAN +# endif // WIN32_LEAN_AND_MEAN +# include +#else // ifdef _WIN32 +# define INVALID_HANDLE_VALUE -1 +#endif // ifdef _WIN32 + +namespace mio { + +// This value may be provided as the `length` parameter to the constructor or +// `map`, in which case a memory mapping of the entire file is created. +enum { map_entire_file = 0 }; + +#ifdef _WIN32 +using file_handle_type = HANDLE; +#else +using file_handle_type = int; +#endif + +// This value represents an invalid file handle type. This can be used to +// determine whether `basic_mmap::file_handle` is valid, for example. +const static file_handle_type invalid_handle = INVALID_HANDLE_VALUE; + +template +struct basic_mmap +{ + using value_type = ByteT; + using size_type = size_t; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = value_type*; + using const_pointer = const value_type*; + using difference_type = std::ptrdiff_t; + using iterator = pointer; + using const_iterator = const_pointer; + using reverse_iterator = std::reverse_iterator; + using const_reverse_iterator = std::reverse_iterator; + using iterator_category = std::random_access_iterator_tag; + using handle_type = file_handle_type; + + static_assert(sizeof(ByteT) == sizeof(char), "ByteT must be the same size as char."); + +private: + // Points to the first requested byte, and not to the actual start of the mapping. + pointer data_ = nullptr; + + // Length, in bytes, requested by user, which may not be the length of the full + // mapping, and the entire length of the full mapping. + size_type length_ = 0; + size_type mapped_length_ = 0; + + // Letting user map a file using both an existing file handle and a path introcudes + // On POSIX, we only need a file handle to create a mapping, while on Windows + // systems the file handle is necessary to retrieve a file mapping handle, but any + // subsequent operations on the mapped region must be done through the latter. + handle_type file_handle_ = INVALID_HANDLE_VALUE; +#ifdef _WIN32 + handle_type file_mapping_handle_ = INVALID_HANDLE_VALUE; +#endif + + // Letting user map a file using both an existing file handle and a path + // introcudes some complexity in that we must not close the file handle if + // user provided it, but we must close it if we obtained it using the + // provided path. For this reason, this flag is used to determine when to + // close file_handle_. + bool is_handle_internal_; + +public: + /** + * The default constructed mmap object is in a non-mapped state, that is, + * any operation that attempts to access nonexistent underlying data will + * result in undefined behaviour/segmentation faults. + */ + basic_mmap() = default; + +#ifdef __cpp_exceptions + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + template + basic_mmap(const String& path, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(path, offset, length, error); + if(error) { throw std::system_error(error); } + } + + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + basic_mmap(const handle_type handle, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(handle, offset, length, error); + if(error) { throw std::system_error(error); } + } +#endif // __cpp_exceptions + + /** + * `basic_mmap` has single-ownership semantics, so transferring ownership + * may only be accomplished by moving the object. + */ + basic_mmap(const basic_mmap&) = delete; + basic_mmap(basic_mmap&&); + basic_mmap& operator=(const basic_mmap&) = delete; + basic_mmap& operator=(basic_mmap&&); + + /** + * If this is a read-write mapping, the destructor invokes sync. Regardless + * of the access mode, unmap is invoked as a final step. + */ + ~basic_mmap(); + + /** + * On UNIX systems 'file_handle' and 'mapping_handle' are the same. On Windows, + * however, a mapped region of a file gets its own handle, which is returned by + * 'mapping_handle'. + */ + handle_type file_handle() const noexcept { return file_handle_; } + handle_type mapping_handle() const noexcept; + + /** Returns whether a valid memory mapping has been created. */ + bool is_open() const noexcept { return file_handle_ != invalid_handle; } + + /** + * Returns true if no mapping was established, that is, conceptually the + * same as though the length that was mapped was 0. This function is + * provided so that this class has Container semantics. + */ + bool empty() const noexcept { return length() == 0; } + + /** Returns true if a mapping was established. */ + bool is_mapped() const noexcept; + + /** + * `size` and `length` both return the logical length, i.e. the number of bytes + * user requested to be mapped, while `mapped_length` returns the actual number of + * bytes that were mapped which is a multiple of the underlying operating system's + * page allocation granularity. + */ + size_type size() const noexcept { return length(); } + size_type length() const noexcept { return length_; } + size_type mapped_length() const noexcept { return mapped_length_; } + + /** + * Returns the offset, relative to the file's start, at which the mapping was + * requested to be created. + */ + size_type offset() const noexcept { return mapped_length_ - length_; } + + /** + * Returns a pointer to the first requested byte, or `nullptr` if no memory mapping + * exists. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > pointer data() noexcept { return data_; } + const_pointer data() const noexcept { return data_; } + + /** + * Returns an iterator to the first requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > iterator begin() noexcept { return data(); } + const_iterator begin() const noexcept { return data(); } + const_iterator cbegin() const noexcept { return data(); } + + /** + * Returns an iterator one past the last requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > iterator end() noexcept { return data() + length(); } + const_iterator end() const noexcept { return data() + length(); } + const_iterator cend() const noexcept { return data() + length(); } + + /** + * Returns a reverse iterator to the last memory mapped byte, if a valid + * memory mapping exists, otherwise this function call is undefined + * behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rbegin() noexcept { return reverse_iterator(end()); } + const_reverse_iterator rbegin() const noexcept + { return const_reverse_iterator(end()); } + const_reverse_iterator crbegin() const noexcept + { return const_reverse_iterator(end()); } + + /** + * Returns a reverse iterator past the first mapped byte, if a valid memory + * mapping exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rend() noexcept { return reverse_iterator(begin()); } + const_reverse_iterator rend() const noexcept + { return const_reverse_iterator(begin()); } + const_reverse_iterator crend() const noexcept + { return const_reverse_iterator(begin()); } + + /** + * Returns a reference to the `i`th byte from the first requested byte (as returned + * by `data`). If this is invoked when no valid memory mapping has been created + * prior to this call, undefined behaviour ensues. + */ + reference operator[](const size_type i) noexcept { return data_[i]; } + const_reference operator[](const size_type i) const noexcept { return data_[i]; } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + template + void map(const String& path, const size_type offset, + const size_type length, std::error_code& error); + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * The entire file is mapped. + */ + template + void map(const String& path, std::error_code& error) + { + map(path, 0, map_entire_file, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is + * unsuccesful, the reason is reported via `error` and the object remains in + * a state as if this function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + void map(const handle_type handle, const size_type offset, + const size_type length, std::error_code& error); + + /** + * Establishes a memory mapping with AccessMode. If the mapping is + * unsuccesful, the reason is reported via `error` and the object remains in + * a state as if this function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * The entire file is mapped. + */ + void map(const handle_type handle, std::error_code& error) + { + map(handle, 0, map_entire_file, error); + } + + /** + * If a valid memory mapping has been created prior to this call, this call + * instructs the kernel to unmap the memory region and disassociate this object + * from the file. + * + * The file handle associated with the file that is mapped is only closed if the + * mapping was created using a file path. If, on the other hand, an existing + * file handle was used to create the mapping, the file handle is not closed. + */ + void unmap(); + + void swap(basic_mmap& other); + + /** Flushes the memory mapped page to disk. Errors are reported via `error`. */ + template + typename std::enable_if::type + sync(std::error_code& error); + + /** + * All operators compare the address of the first byte and size of the two mapped + * regions. + */ + +private: + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > pointer get_mapping_start() noexcept + { + return !data() ? nullptr : data() - offset(); + } + + const_pointer get_mapping_start() const noexcept + { + return !data() ? nullptr : data() - offset(); + } + + /** + * The destructor syncs changes to disk if `AccessMode` is `write`, but not + * if it's `read`, but since the destructor cannot be templated, we need to + * do SFINAE in a dedicated function, where one syncs and the other is a noop. + */ + template + typename std::enable_if::type + conditional_sync(); + template + typename std::enable_if::type conditional_sync(); +}; + +template +bool operator==(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator!=(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator<(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator<=(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator>(const basic_mmap& a, + const basic_mmap& b); + +template +bool operator>=(const basic_mmap& a, + const basic_mmap& b); + +/** + * This is the basis for all read-only mmap objects and should be preferred over + * directly using `basic_mmap`. + */ +template +using basic_mmap_source = basic_mmap; + +/** + * This is the basis for all read-write mmap objects and should be preferred over + * directly using `basic_mmap`. + */ +template +using basic_mmap_sink = basic_mmap; + +/** + * These aliases cover the most common use cases, both representing a raw byte stream + * (either with a char or an unsigned char/uint8_t). + */ +using mmap_source = basic_mmap_source; +using ummap_source = basic_mmap_source; + +using mmap_sink = basic_mmap_sink; +using ummap_sink = basic_mmap_sink; + +/** + * Convenience factory method that constructs a mapping for any `basic_mmap` or + * `basic_mmap` type. + */ +template< + typename MMap, + typename MappingToken +> MMap make_mmap(const MappingToken& token, + int64_t offset, int64_t length, std::error_code& error) +{ + MMap mmap; + mmap.map(token, offset, length, error); + return mmap; +} + +/** + * Convenience factory method. + * + * MappingToken may be a String (`std::string`, `std::string_view`, `const char*`, + * `std::filesystem::path`, `std::vector`, or similar), or a + * `mmap_source::handle_type`. + */ +template +mmap_source make_mmap_source(const MappingToken& token, mmap_source::size_type offset, + mmap_source::size_type length, std::error_code& error) +{ + return make_mmap(token, offset, length, error); +} + +template +mmap_source make_mmap_source(const MappingToken& token, std::error_code& error) +{ + return make_mmap_source(token, 0, map_entire_file, error); +} + +/** + * Convenience factory method. + * + * MappingToken may be a String (`std::string`, `std::string_view`, `const char*`, + * `std::filesystem::path`, `std::vector`, or similar), or a + * `mmap_sink::handle_type`. + */ +template +mmap_sink make_mmap_sink(const MappingToken& token, mmap_sink::size_type offset, + mmap_sink::size_type length, std::error_code& error) +{ + return make_mmap(token, offset, length, error); +} + +template +mmap_sink make_mmap_sink(const MappingToken& token, std::error_code& error) +{ + return make_mmap_sink(token, 0, map_entire_file, error); +} + +} // namespace mio + +// #include "detail/mmap.ipp" +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_BASIC_MMAP_IMPL +#define MIO_BASIC_MMAP_IMPL + +// #include "mio/mmap.hpp" + +// #include "mio/page.hpp" + +// #include "mio/detail/string_util.hpp" +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_STRING_UTIL_HEADER +#define MIO_STRING_UTIL_HEADER + +#include + +namespace mio { +namespace detail { + +template< + typename S, + typename C = typename std::decay::type, + typename = decltype(std::declval().data()), + typename = typename std::enable_if< + std::is_same::value +#ifdef _WIN32 + || std::is_same::value +#endif + >::type +> struct char_type_helper { + using type = typename C::value_type; +}; + +template +struct char_type { + using type = typename char_type_helper::type; +}; + +// TODO: can we avoid this brute force approach? +template<> +struct char_type { + using type = char; +}; + +template<> +struct char_type { + using type = char; +}; + +template +struct char_type { + using type = char; +}; + +template +struct char_type { + using type = char; +}; + +#ifdef _WIN32 +template<> +struct char_type { + using type = wchar_t; +}; + +template<> +struct char_type { + using type = wchar_t; +}; + +template +struct char_type { + using type = wchar_t; +}; + +template +struct char_type { + using type = wchar_t; +}; +#endif // _WIN32 + +template +struct is_c_str_helper +{ + static constexpr bool value = std::is_same< + CharT*, + // TODO: I'm so sorry for this... Can this be made cleaner? + typename std::add_pointer< + typename std::remove_cv< + typename std::remove_pointer< + typename std::decay< + S + >::type + >::type + >::type + >::type + >::value; +}; + +template +struct is_c_str +{ + static constexpr bool value = is_c_str_helper::value; +}; + +#ifdef _WIN32 +template +struct is_c_wstr +{ + static constexpr bool value = is_c_str_helper::value; +}; +#endif // _WIN32 + +template +struct is_c_str_or_c_wstr +{ + static constexpr bool value = is_c_str::value +#ifdef _WIN32 + || is_c_wstr::value +#endif + ; +}; + +template< + typename String, + typename = decltype(std::declval().data()), + typename = typename std::enable_if::value>::type +> const typename char_type::type* c_str(const String& path) +{ + return path.data(); +} + +template< + typename String, + typename = decltype(std::declval().empty()), + typename = typename std::enable_if::value>::type +> bool empty(const String& path) +{ + return path.empty(); +} + +template< + typename String, + typename = typename std::enable_if::value>::type +> const typename char_type::type* c_str(String path) +{ + return path; +} + +template< + typename String, + typename = typename std::enable_if::value>::type +> bool empty(String path) +{ + return !path || (*path == 0); +} + +} // namespace detail +} // namespace mio + +#endif // MIO_STRING_UTIL_HEADER + + +#include + +#ifndef _WIN32 +# include +# include +# include +# include +#endif + +namespace mio { +namespace detail { + +#ifdef _WIN32 +namespace win { + +/** Returns the 4 upper bytes of an 8-byte integer. */ +inline DWORD int64_high(int64_t n) noexcept +{ + return n >> 32; +} + +/** Returns the 4 lower bytes of an 8-byte integer. */ +inline DWORD int64_low(int64_t n) noexcept +{ + return n & 0xffffffff; +} + +template< + typename String, + typename = typename std::enable_if< + std::is_same::type, char>::value + >::type +> file_handle_type open_file_helper(const String& path, const access_mode mode) +{ + return ::CreateFileA(c_str(path), + mode == access_mode::read ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + 0, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + 0); +} + +template +typename std::enable_if< + std::is_same::type, wchar_t>::value, + file_handle_type +>::type open_file_helper(const String& path, const access_mode mode) +{ + return ::CreateFileW(c_str(path), + mode == access_mode::read ? GENERIC_READ : GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + 0, + OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, + 0); +} + +} // win +#endif // _WIN32 + +/** + * Returns the last platform specific system error (errno on POSIX and + * GetLastError on Win) as a `std::error_code`. + */ +inline std::error_code last_error() noexcept +{ + std::error_code error; +#ifdef _WIN32 + error.assign(GetLastError(), std::system_category()); +#else + error.assign(errno, std::system_category()); +#endif + return error; +} + +template +file_handle_type open_file(const String& path, const access_mode mode, + std::error_code& error) +{ + error.clear(); + if(detail::empty(path)) + { + error = std::make_error_code(std::errc::invalid_argument); + return invalid_handle; + } +#ifdef _WIN32 + const auto handle = win::open_file_helper(path, mode); +#else // POSIX + const auto handle = ::open(c_str(path), + mode == access_mode::read ? O_RDONLY : O_RDWR); +#endif + if(handle == invalid_handle) + { + error = detail::last_error(); + } + return handle; +} + +inline size_t query_file_size(file_handle_type handle, std::error_code& error) +{ + error.clear(); +#ifdef _WIN32 + LARGE_INTEGER file_size; + if(::GetFileSizeEx(handle, &file_size) == 0) + { + error = detail::last_error(); + return 0; + } + return static_cast(file_size.QuadPart); +#else // POSIX + struct stat sbuf; + if(::fstat(handle, &sbuf) == -1) + { + error = detail::last_error(); + return 0; + } + return sbuf.st_size; +#endif +} + +struct mmap_context +{ + char* data; + int64_t length; + int64_t mapped_length; +#ifdef _WIN32 + file_handle_type file_mapping_handle; +#endif +}; + +inline mmap_context memory_map(const file_handle_type file_handle, const int64_t offset, + const int64_t length, const access_mode mode, std::error_code& error) +{ + const int64_t aligned_offset = make_offset_page_aligned(offset); + const int64_t length_to_map = offset - aligned_offset + length; +#ifdef _WIN32 + const int64_t max_file_size = offset + length; + const auto file_mapping_handle = ::CreateFileMapping( + file_handle, + 0, + mode == access_mode::read ? PAGE_READONLY : PAGE_READWRITE, + win::int64_high(max_file_size), + win::int64_low(max_file_size), + 0); + if(file_mapping_handle == invalid_handle) + { + error = detail::last_error(); + return {}; + } + char* mapping_start = static_cast(::MapViewOfFile( + file_mapping_handle, + mode == access_mode::read ? FILE_MAP_READ : FILE_MAP_WRITE, + win::int64_high(aligned_offset), + win::int64_low(aligned_offset), + length_to_map)); + if(mapping_start == nullptr) + { + error = detail::last_error(); + return {}; + } +#else // POSIX + char* mapping_start = static_cast(::mmap( + 0, // Don't give hint as to where to map. + length_to_map, + mode == access_mode::read ? PROT_READ : PROT_WRITE, + MAP_SHARED, + file_handle, + aligned_offset)); + if(mapping_start == MAP_FAILED) + { + error = detail::last_error(); + return {}; + } +#endif + mmap_context ctx; + ctx.data = mapping_start + offset - aligned_offset; + ctx.length = length; + ctx.mapped_length = length_to_map; +#ifdef _WIN32 + ctx.file_mapping_handle = file_mapping_handle; +#endif + return ctx; +} + +} // namespace detail + +// -- basic_mmap -- + +template +basic_mmap::~basic_mmap() +{ + conditional_sync(); + unmap(); +} + +template +basic_mmap::basic_mmap(basic_mmap&& other) + : data_(std::move(other.data_)) + , length_(std::move(other.length_)) + , mapped_length_(std::move(other.mapped_length_)) + , file_handle_(std::move(other.file_handle_)) +#ifdef _WIN32 + , file_mapping_handle_(std::move(other.file_mapping_handle_)) +#endif + , is_handle_internal_(std::move(other.is_handle_internal_)) +{ + other.data_ = nullptr; + other.length_ = other.mapped_length_ = 0; + other.file_handle_ = invalid_handle; +#ifdef _WIN32 + other.file_mapping_handle_ = invalid_handle; +#endif +} + +template +basic_mmap& +basic_mmap::operator=(basic_mmap&& other) +{ + if(this != &other) + { + // First the existing mapping needs to be removed. + unmap(); + data_ = std::move(other.data_); + length_ = std::move(other.length_); + mapped_length_ = std::move(other.mapped_length_); + file_handle_ = std::move(other.file_handle_); +#ifdef _WIN32 + file_mapping_handle_ = std::move(other.file_mapping_handle_); +#endif + is_handle_internal_ = std::move(other.is_handle_internal_); + + // The moved from basic_mmap's fields need to be reset, because + // otherwise other's destructor will unmap the same mapping that was + // just moved into this. + other.data_ = nullptr; + other.length_ = other.mapped_length_ = 0; + other.file_handle_ = invalid_handle; +#ifdef _WIN32 + other.file_mapping_handle_ = invalid_handle; +#endif + other.is_handle_internal_ = false; + } + return *this; +} + +template +typename basic_mmap::handle_type +basic_mmap::mapping_handle() const noexcept +{ +#ifdef _WIN32 + return file_mapping_handle_; +#else + return file_handle_; +#endif +} + +template +template +void basic_mmap::map(const String& path, const size_type offset, + const size_type length, std::error_code& error) +{ + error.clear(); + if(detail::empty(path)) + { + error = std::make_error_code(std::errc::invalid_argument); + return; + } + const auto handle = detail::open_file(path, AccessMode, error); + if(error) + { + return; + } + + map(handle, offset, length, error); + // This MUST be after the call to map, as that sets this to true. + if(!error) + { + is_handle_internal_ = true; + } +} + +template +void basic_mmap::map(const handle_type handle, + const size_type offset, const size_type length, std::error_code& error) +{ + error.clear(); + if(handle == invalid_handle) + { + error = std::make_error_code(std::errc::bad_file_descriptor); + return; + } + + const auto file_size = detail::query_file_size(handle, error); + if(error) + { + return; + } + + if(offset + length > file_size) + { + error = std::make_error_code(std::errc::invalid_argument); + return; + } + + const auto ctx = detail::memory_map(handle, offset, + length == map_entire_file ? (file_size - offset) : length, + AccessMode, error); + if(!error) + { + // We must unmap the previous mapping that may have existed prior to this call. + // Note that this must only be invoked after a new mapping has been created in + // order to provide the strong guarantee that, should the new mapping fail, the + // `map` function leaves this instance in a state as though the function had + // never been invoked. + unmap(); + file_handle_ = handle; + is_handle_internal_ = false; + data_ = reinterpret_cast(ctx.data); + length_ = ctx.length; + mapped_length_ = ctx.mapped_length; +#ifdef _WIN32 + file_mapping_handle_ = ctx.file_mapping_handle; +#endif + } +} + +template +template +typename std::enable_if::type +basic_mmap::sync(std::error_code& error) +{ + error.clear(); + if(!is_open()) + { + error = std::make_error_code(std::errc::bad_file_descriptor); + return; + } + + if(data()) + { +#ifdef _WIN32 + if(::FlushViewOfFile(get_mapping_start(), mapped_length_) == 0 + || ::FlushFileBuffers(file_handle_) == 0) +#else // POSIX + if(::msync(get_mapping_start(), mapped_length_, MS_SYNC) != 0) +#endif + { + error = detail::last_error(); + return; + } + } +#ifdef _WIN32 + if(::FlushFileBuffers(file_handle_) == 0) + { + error = detail::last_error(); + } +#endif +} + +template +void basic_mmap::unmap() +{ + if(!is_open()) { return; } + // TODO do we care about errors here? +#ifdef _WIN32 + if(is_mapped()) + { + ::UnmapViewOfFile(get_mapping_start()); + ::CloseHandle(file_mapping_handle_); + } +#else // POSIX + if(data_) { ::munmap(const_cast(get_mapping_start()), mapped_length_); } +#endif + + // If file_handle_ was obtained by our opening it (when map is called with a path, + // rather than an existing file handle), we need to close it, otherwise it must not + // be closed as it may still be used outside this instance. + if(is_handle_internal_) + { +#ifdef _WIN32 + ::CloseHandle(file_handle_); +#else // POSIX + ::close(file_handle_); +#endif + } + + // Reset fields to their default values. + data_ = nullptr; + length_ = mapped_length_ = 0; + file_handle_ = invalid_handle; +#ifdef _WIN32 + file_mapping_handle_ = invalid_handle; +#endif +} + +template +bool basic_mmap::is_mapped() const noexcept +{ +#ifdef _WIN32 + return file_mapping_handle_ != invalid_handle; +#else // POSIX + return is_open(); +#endif +} + +template +void basic_mmap::swap(basic_mmap& other) +{ + if(this != &other) + { + using std::swap; + swap(data_, other.data_); + swap(file_handle_, other.file_handle_); +#ifdef _WIN32 + swap(file_mapping_handle_, other.file_mapping_handle_); +#endif + swap(length_, other.length_); + swap(mapped_length_, other.mapped_length_); + swap(is_handle_internal_, other.is_handle_internal_); + } +} + +template +template +typename std::enable_if::type +basic_mmap::conditional_sync() +{ + // This is invoked from the destructor, so not much we can do about + // failures here. + std::error_code ec; + sync(ec); +} + +template +template +typename std::enable_if::type +basic_mmap::conditional_sync() +{ + // noop +} + +template +bool operator==(const basic_mmap& a, + const basic_mmap& b) +{ + return a.data() == b.data() + && a.size() == b.size(); +} + +template +bool operator!=(const basic_mmap& a, + const basic_mmap& b) +{ + return !(a == b); +} + +template +bool operator<(const basic_mmap& a, + const basic_mmap& b) +{ + if(a.data() == b.data()) { return a.size() < b.size(); } + return a.data() < b.data(); +} + +template +bool operator<=(const basic_mmap& a, + const basic_mmap& b) +{ + return !(a > b); +} + +template +bool operator>(const basic_mmap& a, + const basic_mmap& b) +{ + if(a.data() == b.data()) { return a.size() > b.size(); } + return a.data() > b.data(); +} + +template +bool operator>=(const basic_mmap& a, + const basic_mmap& b) +{ + return !(a < b); +} + +} // namespace mio + +#endif // MIO_BASIC_MMAP_IMPL + + +#endif // MIO_MMAP_HEADER +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_PAGE_HEADER +#define MIO_PAGE_HEADER + +#ifdef _WIN32 +# include +#else +# include +#endif + +namespace mio { + +/** + * This is used by `basic_mmap` to determine whether to create a read-only or + * a read-write memory mapping. + */ +enum class access_mode +{ + read, + write +}; + +/** + * Determines the operating system's page allocation granularity. + * + * On the first call to this function, it invokes the operating system specific syscall + * to determine the page size, caches the value, and returns it. Any subsequent call to + * this function serves the cached value, so no further syscalls are made. + */ +inline size_t page_size() +{ + static const size_t page_size = [] + { +#ifdef _WIN32 + SYSTEM_INFO SystemInfo; + GetSystemInfo(&SystemInfo); + return SystemInfo.dwAllocationGranularity; +#else + return sysconf(_SC_PAGE_SIZE); +#endif + }(); + return page_size; +} + +/** + * Alligns `offset` to the operating's system page size such that it subtracts the + * difference until the nearest page boundary before `offset`, or does nothing if + * `offset` is already page aligned. + */ +inline size_t make_offset_page_aligned(size_t offset) noexcept +{ + const size_t page_size_ = page_size(); + // Use integer division to round down to the nearest page alignment. + return offset / page_size_ * page_size_; +} + +} // namespace mio + +#endif // MIO_PAGE_HEADER +/* Copyright 2017 https://github.com/mandreyel + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of this + * software and associated documentation files (the "Software"), to deal in the Software + * without restriction, including without limitation the rights to use, copy, modify, + * merge, publish, distribute, sublicense, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be included in all copies + * or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A + * PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE + * OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef MIO_SHARED_MMAP_HEADER +#define MIO_SHARED_MMAP_HEADER + +// #include "mio/mmap.hpp" + + +#include // std::error_code +#include // std::shared_ptr + +namespace mio { + +/** + * Exposes (nearly) the same interface as `basic_mmap`, but endowes it with + * `std::shared_ptr` semantics. + * + * This is not the default behaviour of `basic_mmap` to avoid allocating on the heap if + * shared semantics are not required. + */ +template< + access_mode AccessMode, + typename ByteT +> class basic_shared_mmap +{ + using impl_type = basic_mmap; + std::shared_ptr pimpl_; + +public: + using value_type = typename impl_type::value_type; + using size_type = typename impl_type::size_type; + using reference = typename impl_type::reference; + using const_reference = typename impl_type::const_reference; + using pointer = typename impl_type::pointer; + using const_pointer = typename impl_type::const_pointer; + using difference_type = typename impl_type::difference_type; + using iterator = typename impl_type::iterator; + using const_iterator = typename impl_type::const_iterator; + using reverse_iterator = typename impl_type::reverse_iterator; + using const_reverse_iterator = typename impl_type::const_reverse_iterator; + using iterator_category = typename impl_type::iterator_category; + using handle_type = typename impl_type::handle_type; + using mmap_type = impl_type; + + basic_shared_mmap() = default; + basic_shared_mmap(const basic_shared_mmap&) = default; + basic_shared_mmap& operator=(const basic_shared_mmap&) = default; + basic_shared_mmap(basic_shared_mmap&&) = default; + basic_shared_mmap& operator=(basic_shared_mmap&&) = default; + + /** Takes ownership of an existing mmap object. */ + basic_shared_mmap(mmap_type&& mmap) + : pimpl_(std::make_shared(std::move(mmap))) + {} + + /** Takes ownership of an existing mmap object. */ + basic_shared_mmap& operator=(mmap_type&& mmap) + { + pimpl_ = std::make_shared(std::move(mmap)); + return *this; + } + + /** Initializes this object with an already established shared mmap. */ + basic_shared_mmap(std::shared_ptr mmap) : pimpl_(std::move(mmap)) {} + + /** Initializes this object with an already established shared mmap. */ + basic_shared_mmap& operator=(std::shared_ptr mmap) + { + pimpl_ = std::move(mmap); + return *this; + } + +#ifdef __cpp_exceptions + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + template + basic_shared_mmap(const String& path, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(path, offset, length, error); + if(error) { throw std::system_error(error); } + } + + /** + * The same as invoking the `map` function, except any error that may occur + * while establishing the mapping is wrapped in a `std::system_error` and is + * thrown. + */ + basic_shared_mmap(const handle_type handle, const size_type offset = 0, const size_type length = map_entire_file) + { + std::error_code error; + map(handle, offset, length, error); + if(error) { throw std::system_error(error); } + } +#endif // __cpp_exceptions + + /** + * If this is a read-write mapping and the last reference to the mapping, + * the destructor invokes sync. Regardless of the access mode, unmap is + * invoked as a final step. + */ + ~basic_shared_mmap() = default; + + /** Returns the underlying `std::shared_ptr` instance that holds the mmap. */ + std::shared_ptr get_shared_ptr() { return pimpl_; } + + /** + * On UNIX systems 'file_handle' and 'mapping_handle' are the same. On Windows, + * however, a mapped region of a file gets its own handle, which is returned by + * 'mapping_handle'. + */ + handle_type file_handle() const noexcept + { + return pimpl_ ? pimpl_->file_handle() : invalid_handle; + } + + handle_type mapping_handle() const noexcept + { + return pimpl_ ? pimpl_->mapping_handle() : invalid_handle; + } + + /** Returns whether a valid memory mapping has been created. */ + bool is_open() const noexcept { return pimpl_ && pimpl_->is_open(); } + + /** + * Returns true if no mapping was established, that is, conceptually the + * same as though the length that was mapped was 0. This function is + * provided so that this class has Container semantics. + */ + bool empty() const noexcept { return !pimpl_ || pimpl_->empty(); } + + /** + * `size` and `length` both return the logical length, i.e. the number of bytes + * user requested to be mapped, while `mapped_length` returns the actual number of + * bytes that were mapped which is a multiple of the underlying operating system's + * page allocation granularity. + */ + size_type size() const noexcept { return pimpl_ ? pimpl_->length() : 0; } + size_type length() const noexcept { return pimpl_ ? pimpl_->length() : 0; } + size_type mapped_length() const noexcept + { + return pimpl_ ? pimpl_->mapped_length() : 0; + } + + /** + * Returns the offset, relative to the file's start, at which the mapping was + * requested to be created. + */ + size_type offset() const noexcept { return pimpl_ ? pimpl_->offset() : 0; } + + /** + * Returns a pointer to the first requested byte, or `nullptr` if no memory mapping + * exists. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > pointer data() noexcept { return pimpl_->data(); } + const_pointer data() const noexcept { return pimpl_ ? pimpl_->data() : nullptr; } + + /** + * Returns an iterator to the first requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + iterator begin() noexcept { return pimpl_->begin(); } + const_iterator begin() const noexcept { return pimpl_->begin(); } + const_iterator cbegin() const noexcept { return pimpl_->cbegin(); } + + /** + * Returns an iterator one past the last requested byte, if a valid memory mapping + * exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > iterator end() noexcept { return pimpl_->end(); } + const_iterator end() const noexcept { return pimpl_->end(); } + const_iterator cend() const noexcept { return pimpl_->cend(); } + + /** + * Returns a reverse iterator to the last memory mapped byte, if a valid + * memory mapping exists, otherwise this function call is undefined + * behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rbegin() noexcept { return pimpl_->rbegin(); } + const_reverse_iterator rbegin() const noexcept { return pimpl_->rbegin(); } + const_reverse_iterator crbegin() const noexcept { return pimpl_->crbegin(); } + + /** + * Returns a reverse iterator past the first mapped byte, if a valid memory + * mapping exists, otherwise this function call is undefined behaviour. + */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > reverse_iterator rend() noexcept { return pimpl_->rend(); } + const_reverse_iterator rend() const noexcept { return pimpl_->rend(); } + const_reverse_iterator crend() const noexcept { return pimpl_->crend(); } + + /** + * Returns a reference to the `i`th byte from the first requested byte (as returned + * by `data`). If this is invoked when no valid memory mapping has been created + * prior to this call, undefined behaviour ensues. + */ + reference operator[](const size_type i) noexcept { return (*pimpl_)[i]; } + const_reference operator[](const size_type i) const noexcept { return (*pimpl_)[i]; } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + template + void map(const String& path, const size_type offset, + const size_type length, std::error_code& error) + { + map_impl(path, offset, length, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `path`, which must be a path to an existing file, is used to retrieve a file + * handle (which is closed when the object destructs or `unmap` is called), which is + * then used to memory map the requested region. Upon failure, `error` is set to + * indicate the reason and the object remains in an unmapped state. + * + * The entire file is mapped. + */ + template + void map(const String& path, std::error_code& error) + { + map_impl(path, 0, map_entire_file, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * `offset` is the number of bytes, relative to the start of the file, where the + * mapping should begin. When specifying it, there is no need to worry about + * providing a value that is aligned with the operating system's page allocation + * granularity. This is adjusted by the implementation such that the first requested + * byte (as returned by `data` or `begin`), so long as `offset` is valid, will be at + * `offset` from the start of the file. + * + * `length` is the number of bytes to map. It may be `map_entire_file`, in which + * case a mapping of the entire file is created. + */ + void map(const handle_type handle, const size_type offset, + const size_type length, std::error_code& error) + { + map_impl(handle, offset, length, error); + } + + /** + * Establishes a memory mapping with AccessMode. If the mapping is unsuccesful, the + * reason is reported via `error` and the object remains in a state as if this + * function hadn't been called. + * + * `handle`, which must be a valid file handle, which is used to memory map the + * requested region. Upon failure, `error` is set to indicate the reason and the + * object remains in an unmapped state. + * + * The entire file is mapped. + */ + void map(const handle_type handle, std::error_code& error) + { + map_impl(handle, 0, map_entire_file, error); + } + + /** + * If a valid memory mapping has been created prior to this call, this call + * instructs the kernel to unmap the memory region and disassociate this object + * from the file. + * + * The file handle associated with the file that is mapped is only closed if the + * mapping was created using a file path. If, on the other hand, an existing + * file handle was used to create the mapping, the file handle is not closed. + */ + void unmap() { if(pimpl_) pimpl_->unmap(); } + + void swap(basic_shared_mmap& other) { pimpl_.swap(other.pimpl_); } + + /** Flushes the memory mapped page to disk. Errors are reported via `error`. */ + template< + access_mode A = AccessMode, + typename = typename std::enable_if::type + > void sync(std::error_code& error) { if(pimpl_) pimpl_->sync(error); } + + /** All operators compare the underlying `basic_mmap`'s addresses. */ + + friend bool operator==(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ == b.pimpl_; + } + + friend bool operator!=(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return !(a == b); + } + + friend bool operator<(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ < b.pimpl_; + } + + friend bool operator<=(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ <= b.pimpl_; + } + + friend bool operator>(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ > b.pimpl_; + } + + friend bool operator>=(const basic_shared_mmap& a, const basic_shared_mmap& b) + { + return a.pimpl_ >= b.pimpl_; + } + +private: + template + void map_impl(const MappingToken& token, const size_type offset, + const size_type length, std::error_code& error) + { + if(!pimpl_) + { + mmap_type mmap = make_mmap(token, offset, length, error); + if(error) { return; } + pimpl_ = std::make_shared(std::move(mmap)); + } + else + { + pimpl_->map(token, offset, length, error); + } + } +}; + +/** + * This is the basis for all read-only mmap objects and should be preferred over + * directly using basic_shared_mmap. + */ +template +using basic_shared_mmap_source = basic_shared_mmap; + +/** + * This is the basis for all read-write mmap objects and should be preferred over + * directly using basic_shared_mmap. + */ +template +using basic_shared_mmap_sink = basic_shared_mmap; + +/** + * These aliases cover the most common use cases, both representing a raw byte stream + * (either with a char or an unsigned char/uint8_t). + */ +using shared_mmap_source = basic_shared_mmap_source; +using shared_ummap_source = basic_shared_mmap_source; + +using shared_mmap_sink = basic_shared_mmap_sink; +using shared_ummap_sink = basic_shared_mmap_sink; + +} // namespace mio + +#endif // MIO_SHARED_MMAP_HEADER diff --git a/test/test.cpp b/test/test.cpp index 169a6c2e..9c506cfd 100644 --- a/test/test.cpp +++ b/test/test.cpp @@ -4,8 +4,10 @@ #include #include #include +#include #include #include +#include int handle_error(const std::error_code& error) { @@ -29,25 +31,34 @@ int main() std::copy(_path, _path + path_len, path); std::error_code error; // Fill buffer, then write it to file. - std::string buffer(0x4000 - 250, 'M'); + const int file_size = 0x4000 - 250; + std::string buffer(file_size, 0); + std::iota(buffer.begin(), buffer.end(), 1); std::ofstream file(path); file << buffer; file.close(); + const size_t offset = 300; + assert(offset < buffer.size()); + { // Map the region of the file to which buffer was written. mio::mmap_source file_view = mio::make_mmap_source( - path, 0, mio::map_entire_file, error); + path, offset, mio::map_entire_file, error); if(error) { return handle_error(error); } + const size_t mapped_size = buffer.size() - offset; assert(file_view.is_open()); - assert(file_view.size() == buffer.size()); + assert(file_view.size() == mapped_size); // Then verify that mmap's bytes correspond to that of buffer. - for(auto i = 0; i < buffer.size(); ++i) { - if(file_view[i] != buffer[i]) { - std::printf("%ith byte mismatch: expected(%i) <> actual(%i)", - i, buffer[i], file_view[i]); + for(size_t buf_idx = offset, view_idx = 0; + buf_idx < buffer.size() && view_idx < mapped_size; + ++buf_idx, ++view_idx) { + if(file_view[view_idx] != buffer[buf_idx]) { + std::printf("%luth byte mismatch: expected(%d) <> actual(%d)", + buf_idx, buffer[buf_idx], file_view[view_idx]); + std::cout << std::flush; assert(0); } } @@ -57,13 +68,16 @@ int main() assert(!file_view.is_open()); assert(shared_file_view.is_open()); - assert(shared_file_view.size() == buffer.size()); - - // Then verify that mmap's bytes correspond to that of buffer. - for(auto i = 0; i < buffer.size(); ++i) { - if(shared_file_view[i] != buffer[i]) { - std::printf("%ith byte mismatch: expected(%i) <> actual(%i)", - i, buffer[i], shared_file_view[i]); + assert(shared_file_view.size() == mapped_size); + + // Then verify that shared_mmap's bytes correspond to that of buffer. + for(size_t buf_idx = offset, view_idx = 0; + buf_idx < buffer.size() && view_idx < mapped_size; + ++buf_idx, ++view_idx) { + if(shared_file_view[view_idx] != buffer[buf_idx]) { + std::printf("%luth byte mismatch: expected(%d) <> actual(%d)", + buf_idx, buffer[buf_idx], shared_file_view[view_idx]); + std::cout << std::flush; assert(0); } } diff --git a/third_party/LICENSE.md b/third_party/LICENSE.md new file mode 100644 index 00000000..23edca5d --- /dev/null +++ b/third_party/LICENSE.md @@ -0,0 +1,28 @@ +amalgamate.py - Amalgamate C source and header files +Copyright (c) 2012, Erik Edlund + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of Erik Edlund, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/third_party/amalgamate.py b/third_party/amalgamate.py new file mode 100644 index 00000000..174e1c53 --- /dev/null +++ b/third_party/amalgamate.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python +# coding=utf-8 + +# amalgamate.py - Amalgamate C source and header files. +# Copyright (c) 2012, Erik Edlund +# +# Redistribution and use in source and binary forms, with or without modification, +# are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of Erik Edlund, nor the names of its contributors may +# be used to endorse or promote products derived from this software without +# specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import datetime +import json +import os +import re + + +class Amalgamation(object): + + # Prepends self.source_path to file_path if needed. + def actual_path(self, file_path): + if not os.path.isabs(file_path): + file_path = os.path.join(self.source_path, file_path) + return file_path + + # Search included file_path in self.include_paths and + # in source_dir if specified. + def find_included_file(self, file_path, source_dir): + search_dirs = self.include_paths[:] + if source_dir: + search_dirs.insert(0, source_dir) + + for search_dir in search_dirs: + search_path = os.path.join(search_dir, file_path) + if os.path.isfile(self.actual_path(search_path)): + return search_path + return None + + def __init__(self, args): + with open(args.config, 'r') as f: + config = json.loads(f.read()) + for key in config: + setattr(self, key, config[key]) + + self.verbose = args.verbose == "yes" + self.prologue = args.prologue + self.source_path = args.source_path + self.included_files = [] + + # Generate the amalgamation and write it to the target file. + def generate(self): + amalgamation = "" + + if self.prologue: + with open(self.prologue, 'r') as f: + amalgamation += datetime.datetime.now().strftime(f.read()) + + if self.verbose: + print("Config:") + print(" target = {0}".format(self.target)) + print(" working_dir = {0}".format(os.getcwd())) + print(" include_paths = {0}".format(self.include_paths)) + print("Creating amalgamation:") + for file_path in self.sources: + # Do not check the include paths while processing the source + # list, all given source paths must be correct. + # actual_path = self.actual_path(file_path) + print(" - processing \"{0}\"".format(file_path)) + t = TranslationUnit(file_path, self, True) + amalgamation += t.content + + with open(self.target, 'w') as f: + f.write(amalgamation) + + print("...done!\n") + if self.verbose: + print("Files processed: {0}".format(self.sources)) + print("Files included: {0}".format(self.included_files)) + print("") + + +def _is_within(match, matches): + for m in matches: + if match.start() > m.start() and \ + match.end() < m.end(): + return True + return False + + +class TranslationUnit(object): + # // C++ comment. + cpp_comment_pattern = re.compile(r"//.*?\n") + + # /* C comment. */ + c_comment_pattern = re.compile(r"/\*.*?\*/", re.S) + + # "complex \"stri\\\ng\" value". + string_pattern = re.compile("[^']" r'".*?(?<=[^\\])"', re.S) + + # Handle simple include directives. Support for advanced + # directives where macros and defines needs to expanded is + # not a concern right now. + include_pattern = re.compile( + r'#\s*include\s+(<|")(?P.*?)("|>)', re.S) + + # #pragma once + pragma_once_pattern = re.compile(r'#\s*pragma\s+once', re.S) + + # Search for pattern in self.content, add the match to + # contexts if found and update the index accordingly. + def _search_content(self, index, pattern, contexts): + match = pattern.search(self.content, index) + if match: + contexts.append(match) + return match.end() + return index + 2 + + # Return all the skippable contexts, i.e., comments and strings + def _find_skippable_contexts(self): + # Find contexts in the content in which a found include + # directive should not be processed. + skippable_contexts = [] + + # Walk through the content char by char, and try to grab + # skippable contexts using regular expressions when found. + i = 1 + content_len = len(self.content) + while i < content_len: + j = i - 1 + current = self.content[i] + previous = self.content[j] + + if current == '"': + # String value. + i = self._search_content(j, self.string_pattern, + skippable_contexts) + elif current == '*' and previous == '/': + # C style comment. + i = self._search_content(j, self.c_comment_pattern, + skippable_contexts) + elif current == '/' and previous == '/': + # C++ style comment. + i = self._search_content(j, self.cpp_comment_pattern, + skippable_contexts) + else: + # Skip to the next char. + i += 1 + + return skippable_contexts + + # Returns True if the match is within list of other matches + + # Removes pragma once from content + def _process_pragma_once(self): + content_len = len(self.content) + if content_len < len("#include "): + return 0 + + # Find contexts in the content in which a found include + # directive should not be processed. + skippable_contexts = self._find_skippable_contexts() + + pragmas = [] + pragma_once_match = self.pragma_once_pattern.search(self.content) + while pragma_once_match: + if not _is_within(pragma_once_match, skippable_contexts): + pragmas.append(pragma_once_match) + + pragma_once_match = self.pragma_once_pattern.search(self.content, + pragma_once_match.end()) + + # Handle all collected pragma once directives. + prev_end = 0 + tmp_content = '' + for pragma_match in pragmas: + tmp_content += self.content[prev_end:pragma_match.start()] + prev_end = pragma_match.end() + tmp_content += self.content[prev_end:] + self.content = tmp_content + + # Include all trivial #include directives into self.content. + def _process_includes(self): + content_len = len(self.content) + if content_len < len("#include "): + return 0 + + # Find contexts in the content in which a found include + # directive should not be processed. + skippable_contexts = self._find_skippable_contexts() + + # Search for include directives in the content, collect those + # which should be included into the content. + includes = [] + include_match = self.include_pattern.search(self.content) + while include_match: + if not _is_within(include_match, skippable_contexts): + include_path = include_match.group("path") + search_same_dir = include_match.group(1) == '"' + found_included_path = self.amalgamation.find_included_file( + include_path, self.file_dir if search_same_dir else None) + if found_included_path: + includes.append((include_match, found_included_path)) + + include_match = self.include_pattern.search(self.content, + include_match.end()) + + # Handle all collected include directives. + prev_end = 0 + tmp_content = '' + for include in includes: + include_match, found_included_path = include + tmp_content += self.content[prev_end:include_match.start()] + tmp_content += "// {0}\n".format(include_match.group(0)) + if found_included_path not in self.amalgamation.included_files: + t = TranslationUnit(found_included_path, self.amalgamation, False) + tmp_content += t.content + prev_end = include_match.end() + tmp_content += self.content[prev_end:] + self.content = tmp_content + + return len(includes) + + # Make all content processing + def _process(self): + if not self.is_root: + self._process_pragma_once() + self._process_includes() + + def __init__(self, file_path, amalgamation, is_root): + self.file_path = file_path + self.file_dir = os.path.dirname(file_path) + self.amalgamation = amalgamation + self.is_root = is_root + + self.amalgamation.included_files.append(self.file_path) + + actual_path = self.amalgamation.actual_path(file_path) + if not os.path.isfile(actual_path): + raise IOError("File not found: \"{0}\"".format(file_path)) + with open(actual_path, 'r') as f: + self.content = f.read() + self._process() + + +def main(): + description = "Amalgamate C source and header files." + usage = " ".join([ + "amalgamate.py", + "[-v]", + "-c path/to/config.json", + "-s path/to/source/dir", + "[-p path/to/prologue.(c|h)]" + ]) + argsparser = argparse.ArgumentParser( + description=description, usage=usage) + + argsparser.add_argument("-v", "--verbose", dest="verbose", + choices=["yes", "no"], metavar="", help="be verbose") + + argsparser.add_argument("-c", "--config", dest="config", + required=True, metavar="", help="path to a JSON config file") + + argsparser.add_argument("-s", "--source", dest="source_path", + required=True, metavar="", help="source code path") + + argsparser.add_argument("-p", "--prologue", dest="prologue", + required=False, metavar="", help="path to a C prologue file") + + amalgamation = Amalgamation(argsparser.parse_args()) + amalgamation.generate() + + +if __name__ == "__main__": + main() + diff --git a/third_party/config.json b/third_party/config.json new file mode 100644 index 00000000..fa2e71ad --- /dev/null +++ b/third_party/config.json @@ -0,0 +1,11 @@ +{ + "project": "Cross-platform C++11 header-only library for memory mapped file IO", + "target": "../single_include/mio/mio.hpp", + "sources": [ + "../include/mio/mmap.hpp", + "../include/mio/page.hpp", + "../include/mio/shared_mmap.hpp" + ], + "include_paths": ["../include"] +} +