Skip to content

Commit

Permalink
Small changes requested by Hannah's review
Browse files Browse the repository at this point in the history 
Mostly rewording of comments and renaming of types and variables.
  • Loading branch information
@joka921
joka921 committed Mar 23, 2021
1 parent 769ea70 commit 14c158c
Show file tree
Hide file tree
Showing 23 changed files with 363 additions and 353 deletions.
5 changes: 2 additions & 3 deletions src/ServerMain.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,8 +51,7 @@ void printUsage(char* execName) {
<< "The port on which to run the web interface." << endl;
cout << " " << std::setw(20) << "m, memory-for-queries" << std::setw(1)
<< " "
<< "The number of GB that may be used by query (intermediate) results, "
"including the cache"
<< "Limit on the total amount of memory (in GB) that can be used for query processing and caching. If exceeded, query will return with an error, but the engine will not crash."
<< endl;
cout << " " << std::setw(20) << "no-patterns" << std::setw(1) << " "
<< "Disable the use of patterns. This disables ql:has-predicate."
Expand Down Expand Up @@ -86,7 +85,7 @@ int main(int argc, char** argv) {
bool usePatterns = true;
bool enablePatternTrick = true;

size_t memLimit = MAX_MEM_FOR_QUERIES_IN_GB;
size_t memLimit = DEFAULT_MEM_FOR_QUERIES_IN_GB;

optind = 1;
// Process command line arguments.
Expand Down
5 changes: 3 additions & 2 deletions src/SparqlEngineMain.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,9 @@ int main(int argc, char** argv) {
index.addTextFromOnDiskIndex();
}

ad_utility::LimitedAllocator<Id> allocator{
ad_utility::makeAllocationState(MAX_MEM_FOR_QUERIES_IN_GB)};
ad_utility::AllocatorWithLimit<Id> allocator{
ad_utility::makeAllocationMemoryLeftThreadsafeObject(
DEFAULT_MEM_FOR_QUERIES_IN_GB)};

QueryExecutionContext qec(index, engine, &cache, &pinnedSizes, allocator);
if (costFactosFileName.size() > 0) {
Expand Down
5 changes: 3 additions & 2 deletions src/WriteIndexListsMain.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,9 @@ int main(int argc, char** argv) {
Engine engine;
SubtreeCache cache(NOF_SUBTREES_TO_CACHE);
PinnedSizes pinnedSizes;
ad_utility::LimitedAllocator<Id> allocator{
ad_utility::makeAllocationState(MAX_MEM_FOR_QUERIES_IN_GB)};
ad_utility::AllocatorWithLimit<Id> allocator{
ad_utility::makeAllocationMemoryLeftThreadsafeObject(
DEFAULT_MEM_FOR_QUERIES_IN_GB)};
QueryExecutionContext qec(index, engine, &cache, &pinnedSizes, allocator);
ParsedQuery q;
if (!freebase) {
Expand Down
100 changes: 51 additions & 49 deletions src/engine/IdTable.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,26 +12,26 @@
#include <ostream>

#include "../global/Id.h"
#include "../util/LimitedAllocator.h"
#include "../util/AllocatorWithLimit.h"
#include "../util/Log.h"

namespace detail {
// The actual data storage of the Id Tables, basically a wrapper around a
// std::vector<Id>
template <typename Alloc>
template <typename Allocator>
struct IdTableVectorWrapper {
static constexpr bool ManagesStorage = true; // is able to grow/allocate
std::vector<Id, Alloc> _data;
std::vector<Id, Allocator> _data;

IdTableVectorWrapper(Alloc a) : _data{a} {};
IdTableVectorWrapper(Allocator a) : _data{a} {};

// construct from c-style array by copying the content
explicit IdTableVectorWrapper(const Id* const ptr, size_t sz, Alloc a)
explicit IdTableVectorWrapper(const Id* const ptr, size_t sz, Allocator a)
: _data{a} {
_data.assign(ptr, ptr + sz);
}

Alloc getAllocator() const { return _data.get_allocator(); }
Allocator getAllocator() const { return _data.get_allocator(); }

// unified interface to the data
Id* data() noexcept { return _data.data(); }
Expand All @@ -43,28 +43,28 @@ struct IdTableVectorWrapper {
// similar interface to the IdTableVectorWrapper but doesn't own storage, used
// for cheap to copy const views into IdTables
// We still need the allocator information to be able to copy to a owning vector
template <typename Alloc>
template <typename Allocator>
struct IdTableViewWrapper {
static constexpr bool ManagesStorage =
false; // is not able to grow and allocate
const Id* _data;
const size_t _size;
Alloc _alloc;
Allocator _allocator;

IdTableViewWrapper() = delete;

IdTableViewWrapper(const IdTableViewWrapper&) noexcept = default;

// construct as a view into an owning VectorWrapper
explicit IdTableViewWrapper(const IdTableVectorWrapper<Alloc>& rhs) noexcept
explicit IdTableViewWrapper(const IdTableVectorWrapper<Allocator>& rhs) noexcept
: _data(rhs.data()),
_size(rhs._data.size()),
_alloc{rhs.getAllocator()} {}
_allocator{rhs.getAllocator()} {}

// convert to an owning VectorWrapper by making a copy. Explicit since
// expensive
explicit operator IdTableVectorWrapper<Alloc>() const {
return IdTableVectorWrapper(_data, _size, _alloc);
explicit operator IdTableVectorWrapper<Allocator>() const {
return IdTableVectorWrapper(_data, _size, _allocator);
}

// access interface to the data
Expand Down Expand Up @@ -240,41 +240,41 @@ class IdTableIterator {
*
* @tparam COLS The number of columns (> 0 for this implementation)
* @tparam DATA IdTableVectorWrapper or IdTableViewWrapper (data storage that
* @tparam Alloc The Allocator used
* @tparam Allocator The Allocator used
*allows access to an (const) Id* )
**/
template <int COLS, typename DATA, typename Alloc>
template <int COLS, typename DATA, typename Allocator>
class IdTableImpl {
protected:
size_t _size = 0;
size_t _capacity = 0;
DATA _data; // something that lets us access a (const) Id*, might or might
// not own its storage

template <int INCOLS, typename INDATA, typename>
template <int INCOLS, typename INDATA, typename INALLOCATOR>
friend class IdTableImpl; // make the conversions work from static to dynamic
// etc.

// these constructors are protected so they can only be used by the
// moveToStatic and moveToStaticView functions of the IdTableStatic class
template <int INCOLS, typename INDATA,
typename = std::enable_if_t<INCOLS == 0 || INCOLS == COLS>>
IdTableImpl(const IdTableImpl<INCOLS, INDATA, Alloc>& o)
IdTableImpl(const IdTableImpl<INCOLS, INDATA, Allocator>& o)
: _size(o._size), _capacity(o._capacity), _data(DATA(o._data)) {
assert(cols() == o.cols());
}

template <int INCOLS, typename INDATA,
typename = std::enable_if_t<INCOLS == 0 || INCOLS == COLS>>
IdTableImpl(IdTableImpl<INCOLS, INDATA, Alloc>&& o) noexcept
IdTableImpl(IdTableImpl<INCOLS, INDATA, Allocator>&& o) noexcept
: _size(std::move(o._size)),
_capacity(std::move(o._capacity)),
_data(std::move(o._data)) {
assert(cols() == o.cols());
}

public:
IdTableImpl(Alloc al) : _data{al} {};
IdTableImpl(Allocator allocator) : _data{allocator} {};

using const_row_type = const std::array<Id, COLS>;
using row_type = const std::array<Id, COLS>;
Expand Down Expand Up @@ -638,8 +638,8 @@ class IdTableDynamicIterator {

// Common interface specialication for the dynamic (number of columns only known
// at runtime) IdTable
template <typename DATA, typename Alloc>
class IdTableImpl<0, DATA, Alloc> {
template <typename DATA, typename Allocator>
class IdTableImpl<0, DATA, Allocator> {
template <int INCOLS, typename INDATA, typename INALLOC>
friend class IdTableImpl;

Expand All @@ -650,18 +650,18 @@ class IdTableImpl<0, DATA, Alloc> {
size_t _cols = 0;

public:
IdTableImpl(Alloc al) : _data{al} {};
IdTableImpl(Allocator al) : _data{al} {};

protected:
template <int INCOLS, typename INDATA>
explicit IdTableImpl(const IdTableImpl<INCOLS, INDATA, Alloc>& o)
explicit IdTableImpl(const IdTableImpl<INCOLS, INDATA, Allocator>& o)
: _size(o._size),
_capacity(o._capacity),
_data(o._data),
_cols(o._cols) {}

template <int INCOLS, typename INDATA>
explicit IdTableImpl(IdTableImpl<INCOLS, INDATA, Alloc>&& o)
explicit IdTableImpl(IdTableImpl<INCOLS, INDATA, Allocator>&& o)
: _size(o._size),
_capacity(o._capacity),
_data(std::move(o._data)),
Expand Down Expand Up @@ -704,16 +704,16 @@ class IdTableImpl<0, DATA, Alloc> {
* @tparam COLS The number of Columns, 0 means "dynamic (const but runtime)
* number of columns)
* @tparam DATA A data Accessor like IdTableVectorWrapper or IdTableViewWrapper
* @tparam Alloc An allocator type
* @tparam Allocator An allocator type
*/
template <int COLS, typename DATA, typename Alloc>
class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
template <int COLS, typename DATA, typename Allocator>
class IdTableTemplated : private IdTableImpl<COLS, DATA, Allocator> {
// Make all other instantiations of this template friends of this.
template <int, typename, typename>
friend class IdTableTemplated;

protected:
using Base = IdTableImpl<COLS, DATA, Alloc>;
using Base = IdTableImpl<COLS, DATA, Allocator>;
// make stuff from the templated base class accessible
using Base::_capacity;
using Base::_cols;
Expand Down Expand Up @@ -742,9 +742,9 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
using Base::cols;
using Base::setCols;

IdTableTemplated(Alloc al = Alloc{}) : Base{al} {};
IdTableTemplated(Allocator al = Allocator{}) : Base{al} {};

IdTableTemplated(size_t cols, Alloc al = Alloc{}) : Base{al} {
IdTableTemplated(size_t cols, Allocator al = Allocator{}) : Base{al} {
setCols(cols);
}

Expand All @@ -759,19 +759,19 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
// move assignment
IdTableTemplated& operator=(IdTableTemplated&& other) noexcept = default;

Alloc getAllocator() const { return _data.getAllocator(); }
Allocator getAllocator() const { return _data.getAllocator(); }

protected:
// internally convert between "dynamic" and "static" mode and possibly
// convert from Owning to non owning storage, is used in the conversion
// functions Base class asserts that number of columns match.
template <int INCOLS, typename INDATA>
explicit IdTableTemplated(const IdTableTemplated<INCOLS, INDATA, Alloc>& o)
explicit IdTableTemplated(const IdTableTemplated<INCOLS, INDATA, Allocator>& o)
: Base(o) {}

template <int INCOLS, typename INDATA>
explicit IdTableTemplated(
IdTableTemplated<INCOLS, INDATA, Alloc>&& o) noexcept
IdTableTemplated<INCOLS, INDATA, Allocator>&& o) noexcept
: Base(std::move(o)) {}

public:
Expand Down Expand Up @@ -901,7 +901,7 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {

template <typename INDATA, bool ManagesStorage = DATA::ManagesStorage,
typename = std::enable_if_t<ManagesStorage>>
void push_back(const IdTableTemplated<COLS, INDATA, Alloc>& init,
void push_back(const IdTableTemplated<COLS, INDATA, Allocator>& init,
size_t row) {
assert(init._cols == _cols);
if (_size + 1 >= _capacity) {
Expand Down Expand Up @@ -1014,8 +1014,8 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
**/
template <int NEW_COLS, bool ManagesStorage = DATA::ManagesStorage,
typename = std::enable_if_t<ManagesStorage>>
IdTableTemplated<NEW_COLS, DATA, Alloc> moveToStatic() {
return IdTableTemplated<NEW_COLS, DATA, Alloc>(
IdTableTemplated<NEW_COLS, DATA, Allocator> moveToStatic() {
return IdTableTemplated<NEW_COLS, DATA, Allocator>(
std::move(*this)); // let the private conversions do all the work
};

Expand All @@ -1026,8 +1026,8 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
**/
template <bool ManagesStorage = DATA::ManagesStorage,
typename = std::enable_if_t<ManagesStorage>>
IdTableTemplated<0, DATA, Alloc> moveToDynamic() {
return IdTableTemplated<0, DATA, Alloc>(std::move(*this));
IdTableTemplated<0, DATA, Allocator> moveToDynamic() {
return IdTableTemplated<0, DATA, Allocator>(std::move(*this));
};

/**
Expand All @@ -1037,10 +1037,10 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
* columns this table already has or an assertion will fail
*/
template <int NEW_COLS>
const IdTableTemplated<NEW_COLS, IdTableViewWrapper<Alloc>, Alloc>
const IdTableTemplated<NEW_COLS, IdTableViewWrapper<Allocator>, Allocator>
asStaticView() const {
return IdTableTemplated<NEW_COLS, IdTableViewWrapper<Alloc>, Alloc>(
*static_cast<const IdTableTemplated<COLS, DATA, Alloc>*>(this));
return IdTableTemplated<NEW_COLS, IdTableViewWrapper<Allocator>, Allocator>(
*static_cast<const IdTableTemplated<COLS, DATA, Allocator>*>(this));
};

/**
Expand All @@ -1052,9 +1052,9 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
*/
template <bool ManagesStorage = DATA::ManagesStorage,
typename = std::enable_if_t<!ManagesStorage>>
const IdTableTemplated<COLS, IdTableVectorWrapper<Alloc>, Alloc> clone()
const IdTableTemplated<COLS, IdTableVectorWrapper<Allocator>, Allocator> clone()
const {
return IdTableTemplated<COLS, IdTableVectorWrapper<Alloc>, Alloc>(*this);
return IdTableTemplated<COLS, IdTableVectorWrapper<Allocator>, Allocator>(*this);
};

// Size access
Expand Down Expand Up @@ -1084,7 +1084,7 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {

// Support for ostreams
friend std::ostream& operator<<(
std::ostream& out, const IdTableTemplated<COLS, DATA, Alloc>& table) {
std::ostream& out, const IdTableTemplated<COLS, DATA, Allocator>& table) {
out << "IdTable(" << ((void*)table.data()) << ") with " << table.size()
<< " rows and " << table.cols() << " columns" << std::endl;
for (size_t row = 0; row < table.size(); row++) {
Expand All @@ -1102,15 +1102,17 @@ class IdTableTemplated : private IdTableImpl<COLS, DATA, Alloc> {
/// The general IdTable class. Can be modified and owns its data. If COLS > 0,
/// COLS specifies the compile-time number of columns COLS == 0 means "runtime
/// number of cols"
template <int COLS, typename Alloc = ad_utility::LimitedAllocator<Id>>
template <int COLS, typename Allocator = ad_utility::AllocatorWithLimit<Id>>
using IdTableStatic =
detail::IdTableTemplated<COLS, detail::IdTableVectorWrapper<Alloc>, Alloc>;
detail::IdTableTemplated<COLS, detail::IdTableVectorWrapper<Allocator>,
Allocator>;

// the "runtime number of cols" variant
// template <typename Alloc = ad_utility::LimitedAllocator<Id>>
using IdTable = IdTableStatic<0, ad_utility::LimitedAllocator<Id>>;
// template <typename Allocator = ad_utility::AllocatorWithLimit<Id>>
using IdTable = IdTableStatic<0, ad_utility::AllocatorWithLimit<Id>>;

/// A constant view into an IdTable that does not own its data
template <int COLS, typename Alloc = ad_utility::LimitedAllocator<Id>>
template <int COLS, typename Allocator = ad_utility::AllocatorWithLimit<Id>>
using IdTableView =
detail::IdTableTemplated<COLS, detail::IdTableViewWrapper<Alloc>, Alloc>;
detail::IdTableTemplated<COLS, detail::IdTableViewWrapper<Allocator>,
Allocator>;
2 changes: 1 addition & 1 deletion src/engine/Operation.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ shared_ptr<const ResultTable> Operation::getResult(bool isRoot) {

if (newResult) {
LOG(TRACE) << "Not in the cache, need to compute result" << endl;
LOG(DEBUG) << "Free Megabytes in the limit before computing the result:"
LOG(DEBUG) << "Available memory (in MB) before operation: "
<< (_executionContext->getAllocator().numFreeBytes() >> 20)
<< std::endl;
// Passing the raw pointer here is ok as the result shared_ptr remains
Expand Down
12 changes: 6 additions & 6 deletions src/engine/QueryExecutionContext.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ using std::string;
using std::vector;

struct CacheValue {
CacheValue(ad_utility::LimitedAllocator<Id> alloc)
: _resTable(std::make_shared<ResultTable>(std::move(alloc))),
CacheValue(ad_utility::AllocatorWithLimit<Id> allocator)
: _resTable(std::make_shared<ResultTable>(std::move(allocator))),
_runtimeInfo() {}
std::shared_ptr<ResultTable> _resTable;
RuntimeInformation _runtimeInfo;
Expand All @@ -45,7 +45,7 @@ class QueryExecutionContext {
QueryExecutionContext(const Index& index, const Engine& engine,
SubtreeCache* const cache,
PinnedSizes* const pinnedSizes,
ad_utility::LimitedAllocator<Id> alloc,
ad_utility::AllocatorWithLimit<Id> allocator,
const bool pinSubtrees = false,
const bool pinResult = false)
: _pinSubtrees(pinSubtrees),
Expand All @@ -54,7 +54,7 @@ class QueryExecutionContext {
_engine(engine),
_subtreeCache(cache),
_pinnedSizes(pinnedSizes),
_alloc(std::move(alloc)),
_alloc(std::move(allocator)),
_costFactors() {}

SubtreeCache& getQueryTreeCache() { return *_subtreeCache; }
Expand All @@ -75,7 +75,7 @@ class QueryExecutionContext {
return _costFactors.getCostFactor(key);
};

ad_utility::LimitedAllocator<Id> getAllocator() { return _alloc; }
ad_utility::AllocatorWithLimit<Id> getAllocator() { return _alloc; }

const bool _pinSubtrees;
const bool _pinResult;
Expand All @@ -86,6 +86,6 @@ class QueryExecutionContext {
SubtreeCache* const _subtreeCache;
PinnedSizes* const _pinnedSizes;
// allocators are copied but hold shared state
ad_utility::LimitedAllocator<Id> _alloc;
ad_utility::AllocatorWithLimit<Id> _alloc;
QueryPlanningCostFactors _costFactors;
};

0 comments on commit 14c158c

Please sign in to comment.