Skip to content

Commit

Permalink
mmap_vec (#208)
Browse files Browse the repository at this point in the history
* mmap_vec

* silence warning

* fix bitvec fuzzing

* upgrade actions/checkout@v3 to v4

* fuzz all

* use clang 17

* gcc 12
  • Loading branch information
felixguendling authored Mar 16, 2024
1 parent f67b3bf commit 3f8c440
Show file tree
Hide file tree
Showing 8 changed files with 263 additions and 38 deletions.
20 changes: 10 additions & 10 deletions .github/actions/install-llvm/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,20 @@ inputs:
version:
description: 'LLVM version'
required: false
default: 16
default: 17
packages:
description: 'Packages to install'
required: false
default: >
clang-16
clang-tools-16
clang-format-16
clang-tidy-16
libc++-16-dev
libc++abi-16-dev
libclang-common-16-dev
libclang-rt-16-dev
libunwind-16-dev
clang-17
clang-tools-17
clang-format-17
clang-tidy-17
libc++-17-dev
libc++abi-17-dev
libclang-common-17-dev
libclang-rt-17-dev
libunwind-17-dev
runs:
using: "composite"
steps:
Expand Down
61 changes: 37 additions & 24 deletions .github/workflows/linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,15 @@ jobs:
formatting:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- name: Install clang-format
uses: ./.github/actions/install-llvm
with:
packages: clang-format-16
packages: clang-format-17

- name: Format files
run: find include test -type f -a \( -name "*.cc" -o -name "*.h" \) -print0 | xargs -0 clang-format-16 -i
run: find include test -type f -a \( -name "*.cc" -o -name "*.h" \) -print0 | xargs -0 clang-format-17 -i

- name: Check for differences
run: |
Expand All @@ -36,37 +36,37 @@ jobs:
strategy:
matrix:
config:
- name: GCC 11 Release
cxx: g++-11
cc: gcc-11
- name: GCC 12 Release
cxx: g++-12
cc: gcc-12
mode: Release
- name: GCC 11 Debug
cxx: g++-11
cc: gcc-11
- name: GCC 12 Debug
cxx: g++-12
cc: gcc-12
mode: Debug
valgrind: true
- name: Clang 16 Release
cxx: clang++-16
cc: clang-16
- name: Clang 17 Release
cxx: clang++-17
cc: clang-17
mode: Release
cxxflags: -stdlib=libc++
ldflags: -lc++abi
- name: Clang 16 Debug
cxx: clang++-16
cc: clang-16
- name: Clang 17 Debug
cxx: clang++-17
cc: clang-17
mode: Debug
fuzz: true
- key: GCC 11 Sanitizer
cxx: g++-11
cc: gcc-11
- key: Clang 17 Sanitizer
cxx: clang++-17
cc: clang-17
mode: Debug
cflags: -fsanitize=address,undefined -fno-omit-frame-pointer
cxxflags: -fsanitize=address,undefined -fno-omit-frame-pointer

env:
UBSAN_OPTIONS: halt_on_error=1:abort_on_error=1
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4

- name: Install Ninja
env:
Expand All @@ -77,7 +77,7 @@ jobs:
- name: Install Clang
uses: ./.github/actions/install-llvm
with:
packages: libstdc++-12-dev libc++-16-dev libc++abi-16-dev clang-tidy-16 libunwind-16-dev llvm-16 libfuzzer-16-dev llvm-16-dev libclang-rt-16-dev
packages: libstdc++-12-dev libc++-17-dev libc++abi-17-dev clang-tidy-17 libunwind-17-dev llvm-17 libfuzzer-17-dev llvm-17-dev libclang-rt-17-dev

# ==== BUILD ====
- name: CMake
Expand All @@ -91,7 +91,7 @@ jobs:
-DCMAKE_CXX_LINKER_FLAGS=${{ matrix.config.ldflags }}" \
-DCMAKE_CXX_EXE_LINKER_FLAGS="${{ matrix.config.ldflags }} \
-DCMAKE_BUILD_TYPE=${{ matrix.config.mode }} \
-DCISTA_ZERO_OUT=${{ matrix.config.mode == 'Debug' && matrix.config.cc == 'gcc-11' }}
-DCISTA_ZERO_OUT=${{ matrix.config.mode == 'Debug' && matrix.config.cc == 'gcc-12' }}
- name: Build
run: cmake --build build --target cista-test cista-test-single-header

Expand Down Expand Up @@ -119,19 +119,32 @@ jobs:
# ==== FUZZ ====
- name: Fuzzing
if: matrix.config.fuzz
run: cmake --build build --target cista-fuzz
run: |
cmake --build build --target \
cista-fuzz-bitset_verification \
cista-fuzz-bitvec_verification \
cista-fuzz-graph \
cista-fuzz-hash_map_verification \
cista-fuzz-hash_set \
cista-fuzz-multimap_verification
./build/cista-fuzz-hash_set -max_total_time=120
./build/cista-fuzz-hash_map_verification -max_total_time=120
./build/cista-fuzz-bitset_verification -max_total_time=120
./build/cista-fuzz-bitvec_verification -max_total_time=120
./build/cista-fuzz-graph -max_total_time=120
./build/cista-fuzz-multimap_verification -max_total_time=120
# ==== DISTRIBUTION ====
- name: Upload Distribution
if: matrix.config.mode == 'Release' && matrix.config.cc == 'gcc-11'
if: matrix.config.mode == 'Release' && matrix.config.cc == 'gcc-12'
uses: actions/upload-artifact@v1
with:
name: cista.h
path: build/cista.h

# ==== RELEASE ====
- name: Upload Release
if: github.event.action == 'published' && matrix.config.mode == 'Release' && matrix.config.cc == 'gcc-11'
if: github.event.action == 'published' && matrix.config.mode == 'Release' && matrix.config.cc == 'gcc-12'
uses: actions/upload-release-asset@v1.0.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ jobs:
CC: cl.exe

steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
- uses: ilammy/msvc-dev-cmd@v1

- name: Install ninja
Expand Down
1 change: 0 additions & 1 deletion CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,6 @@ add_custom_command(
COMMAND uniter
${CMAKE_CURRENT_SOURCE_DIR}/LICENSE
${CMAKE_CURRENT_SOURCE_DIR}/include
${CMAKE_CURRENT_SOURCE_DIR}/include/cista/mmap.h
${CMAKE_CURRENT_SOURCE_DIR}/include/cista/serialization.h
${CMAKE_CURRENT_SOURCE_DIR}/include/cista/reflection/comparable.h
${CMAKE_CURRENT_SOURCE_DIR}/include/cista/reflection/printable.h
Expand Down
7 changes: 5 additions & 2 deletions fuzz/bitvec_verification.cc
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ bool any(std::vector<bool> const& v) {
template <typename T>
bool bitvec_lt(T const& x, T const& y) {
assert(x.size() == y.size());
if (x.size() == 0U) {
return false;
}
for (std::size_t i = 0U; i != x.size(); ++i) {
auto const j = static_cast<std::size_t>(x.size() - i - 1U);
if (x[i] ^ y[i]) {
Expand Down Expand Up @@ -117,8 +120,8 @@ struct test_set {
};

if (bitvec_lt(ref1, ref2) != bitvec_lt(uut1, uut2)) {
std::cerr << "uut1 < uut2 => " << (uut1 < uut2) << "\n"
<< "ref1 < ref2 => " << (ref1 < ref2) << "\n";
std::cerr << "uut1 < uut2 => " << bitvec_lt(uut1, uut2) << "\n"
<< "ref1 < ref2 => " << bitvec_lt(ref1, ref2) << "\n";
print("fail on <");
abort();
}
Expand Down
1 change: 1 addition & 0 deletions include/cista/containers.h
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "cista/containers/fws_multimap.h"
#include "cista/containers/hash_map.h"
#include "cista/containers/hash_set.h"
#include "cista/containers/mmap_vec.h"
#include "cista/containers/mutable_fws_multimap.h"
#include "cista/containers/nvec.h"
#include "cista/containers/optional.h"
Expand Down
173 changes: 173 additions & 0 deletions include/cista/containers/mmap_vec.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
#pragma once

#include <cassert>

#include "cista/mmap.h"
#include "cista/strong.h"

namespace cista {

template <typename T, typename Key = std::size_t>
struct basic_mmap_vec {
using size_type = base_t<Key>;
using difference_type = std::ptrdiff_t;
using access_type = Key;
using reference = T&;
using const_reference = T const&;
using pointer = T*;
using const_pointer = T*;
using value_type = T;
using iterator = T*;
using const_iterator = T const*;

static_assert(std::is_trivially_copyable_v<T>);

explicit basic_mmap_vec(cista::mmap mmap)
: mmap_{std::move(mmap)}, used_size_{mmap_.size() / sizeof(T)} {}

void push_back(T const& t) {
++used_size_;
mmap_.resize(sizeof(T) * used_size_);
(*this)[Key{used_size_ - 1U}] = t;
}

template <typename... Args>
T& emplace_back(Args&&... el) {
reserve(used_size_ + 1U);
new (data() + used_size_) T{std::forward<Args>(el)...};
T* ptr = data() + used_size_;
++used_size_;
return *ptr;
}

std::size_t size() const { return used_size_; }

T const* data() const noexcept { return begin(); }
T* data() noexcept { return begin(); }
T const* begin() const noexcept {
return reinterpret_cast<T const*>(mmap_.data());
}
T const* end() const noexcept { return begin() + used_size_; } // NOLINT
T const* cbegin() const noexcept { return begin(); }
T const* cend() const noexcept { return begin() + used_size_; } // NOLINT
T* begin() noexcept { return reinterpret_cast<T*>(mmap_.data()); }
T* end() noexcept { return begin() + used_size_; } // NOLINT

bool empty() const noexcept { return size() == 0U; }

T const& operator[](access_type const index) const noexcept {
assert(index < used_size_);
return begin()[to_idx(index)];
}

T& operator[](access_type const index) noexcept {
assert(used_size_);
return begin()[to_idx(index)];
}

void reserve(std::size_t const size) { mmap_.resize(size * sizeof(T)); }

void resize(std::size_t const size) {
used_size_ = size;
mmap_.resize(size);
}

template <typename It>
void set(It begin_it, It end_it) {
using diff_t =
std::common_type_t<typename std::iterator_traits<It>::difference_type,
size_type>;
auto const range_size = std::distance(begin_it, end_it);
verify(range_size >= 0 &&
static_cast<diff_t>(range_size) <=
static_cast<diff_t>(std::numeric_limits<size_type>::max()),
"cista::vector::set: invalid range");

reserve(static_cast<size_type>(range_size));

auto copy_source = begin_it;
auto copy_target = data();
for (; copy_source != end_it; ++copy_source, ++copy_target) {
new (copy_target) T{std::forward<decltype(*copy_source)>(*copy_source)};
}

used_size_ = static_cast<size_type>(range_size);
}

template <typename Arg>
T* insert(T* it, Arg&& el) {
auto const old_offset = std::distance(begin(), it);
auto const old_size = used_size_;

reserve(used_size_ + 1);
new (data() + used_size_) T{std::forward<Arg&&>(el)};
++used_size_;

return std::rotate(begin() + old_offset, begin() + old_size, end());
}

template <class InputIt>
T* insert(T* pos, InputIt first, InputIt last, std::input_iterator_tag) {
auto const old_offset = std::distance(begin(), pos);
auto const old_size = used_size_;

for (; !(first == last); ++first) {
reserve(used_size_ + 1);
new (data() + used_size_) T{std::forward<decltype(*first)>(*first)};
++used_size_;
}

return std::rotate(begin() + old_offset, begin() + old_size, end());
}

template <class FwdIt>
T* insert(T* pos, FwdIt first, FwdIt last, std::forward_iterator_tag) {
if (empty()) {
set(first, last);
return begin();
}

auto const pos_idx = pos - begin();
auto const new_count = static_cast<size_type>(std::distance(first, last));
reserve(used_size_ + new_count);
pos = begin() + pos_idx;

for (auto src_last = end() - 1, dest_last = end() + new_count - 1;
!(src_last == pos - 1); --src_last, --dest_last) {
if (dest_last >= end()) {
new (dest_last) T(std::move(*src_last));
} else {
*dest_last = std::move(*src_last);
}
}

for (auto insert_ptr = pos; !(first == last); ++first, ++insert_ptr) {
if (insert_ptr >= end()) {
new (insert_ptr) T(std::forward<decltype(*first)>(*first));
} else {
*insert_ptr = std::forward<decltype(*first)>(*first);
}
}

used_size_ += new_count;

return pos;
}

template <class It>
T* insert(T* pos, It first, It last) {
return insert(pos, first, last,
typename std::iterator_traits<It>::iterator_category());
}

cista::mmap mmap_;
std::size_t used_size_{0U};
};

template <typename T>
using mmap_vec = basic_mmap_vec<T>;

template <typename Key, typename T>
using mmap_vec_map = basic_mmap_vec<Key, T>;

} // namespace cista
Loading

0 comments on commit 3f8c440

Please sign in to comment.