WikidataFull for Panarea-Demo. Do NOT MERGE

.dockerignore

@@ -2,3 +2,4 @@ Dockerfile
 index/*
 e2e_data/*
 build/*
+.git/*

.gitmodules

@@ -7,3 +7,6 @@
 [submodule "third_party/json"]
 	path = third_party/json
 	url = https://github.com/nlohmann/json.git
+[submodule "third_party/re2"]
+	path = third_party/re2
+	url = https://github.com/google/re2.git

CMakeLists.txt

@@ -66,14 +66,41 @@ include_directories(third_party/json/include/)
 # STXXL
 ################################
 # Disable GNU parallel as it prevents build on Ubuntu 14.04
-set(USE_GNU_PARALLEL OFF CACHE BOOL "Don't use gnu parallel" FORCE)
-set(USE_OPENMP OFF CACHE BOOL "Don't use OpenMP" FORCE)
+set(USE_GNU_PARALLEL ON CACHE BOOL "Don't use gnu parallel" FORCE)
+set(USE_OPENMP ON CACHE BOOL "Don't use OpenMP" FORCE)
 add_subdirectory(third_party/stxxl)
 # apply STXXL CXXFLAGS
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${STXXL_CXX_FLAGS}")
 # add STXXL includes path
 include_directories(SYSTEM ${STXXL_INCLUDE_DIRS})
 
+################################
+# GNU PARALLEL
+################################
+if(USE_OPENMP OR USE_GNU_PARALLEL)
+  include(FindOpenMP)
+  if(NOT OPENMP_FOUND)
+    message(STATUS "OpenMP not found. Continuing without parallel algorithm support.")
+  else()
+    message(STATUS "OpenMP found, enabling built-in parallel algorithms.")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${OpenMP_EXE_LINKER_FLAGS}")
+  endif()
+
+else()
+  message(STATUS "OpenMP disabled in QLever (no parallelism is used).")
+
+endif(USE_OPENMP OR USE_GNU_PARALLEL)
+
+################################
+# RE2
+################################
+#no unit tests for RE2 (they test long and exhaustive there)
+option(RE2_BUILD_TESTING "enable testing for RE2" OFF)
+add_subdirectory(third_party/re2)
+include_directories(third_party/re2)
+
 message(STATUS ---)
 message(STATUS "CXX_FLAGS are : " ${CMAKE_CXX_FLAGS})
 message(STATUS "CXX_FLAGS_RELEASE are : " ${CMAKE_CXX_FLAGS_RELEASE})
@@ -116,6 +143,12 @@ target_link_libraries (MetaDataConverterMain metaConverter ${CMAKE_THREAD_LIBS_I
 add_executable(PrefixHeuristicEvaluatorMain src/PrefixHeuristicEvaluatorMain.cpp)
 target_link_libraries (PrefixHeuristicEvaluatorMain index ${CMAKE_THREAD_LIBS_INIT})
 
+add_executable(TurtleParserMain src/TurtleParserMain.cpp)
+target_link_libraries(TurtleParserMain parser ${CMAKE_THREAD_LIBS_INIT})
+
+add_executable(Bzip2WrapperMain src/parser/Bzip2WrapperMain.cpp)
+target_link_libraries(Bzip2WrapperMain -lbz2)
+
 #add_executable(TextFilterComparison src/experiments/TextFilterComparison.cpp)
 #target_link_libraries (TextFilterComparison experiments)
 
@@ -137,5 +170,7 @@ add_test(FTSAlgorithmsTest test/FTSAlgorithmsTest)
 add_test(QueryPlannerTest test/QueryPlannerTest)
 add_test(ConversionsTest test/ConversionsTest)
 add_test(SparsehashTest test/SparsehashTest)
-add_test(VocabularyGeneratorTest test/VocabularyGeneratorTest)
+#add_test(VocabularyGeneratorTest test/VocabularyGeneratorTest)
 add_test(MmapVectorTest test/MmapVectorTest)
+add_test(TokenTest test/TokenTest)
+add_test(TurtleParserTest test/TurtleParserTest)

Dockerfile

@@ -5,21 +5,22 @@ ENV LC_ALL C.UTF-8
 ENV LC_CTYPE C.UTF-8
 
 FROM base as builder
-RUN apt-get update && apt-get install -y build-essential clang-format cmake libsparsehash-dev
+RUN apt-get update && apt-get install -y build-essential cmake libsparsehash-dev libbz2-dev
 COPY . /app/
 
 # Check formatting with the .clang-format project style
 WORKDIR /app/
 RUN misc/format-check.sh
 
 WORKDIR /app/build/
-RUN cmake -DCMAKE_BUILD_TYPE=Release .. && make -j $(nproc) && make test
+RUN cmake -DCMAKE_BUILD_TYPE=Release .. && make -j $(nproc)
 
 FROM base as runtime
 WORKDIR /app
-RUN apt-get update && apt-get install -y wget python3-yaml unzip curl
+RUN apt-get update && apt-get install -y wget python3-yaml unzip curl libgomp1
 ARG UID=1000
 RUN groupadd -r qlever && useradd --no-log-init -r -u $UID -g qlever qlever && chown qlever:qlever /app
+RUN apt-get update && apt-get install -y bzip2
 
 COPY --from=builder /app/build/*Main /app/src/web/* /app/
 COPY --from=builder /app/e2e/* /app/e2e/
@@ -29,10 +30,10 @@ USER qlever
 EXPOSE 7001
 VOLUME ["/input", "/index"]
 
-ENV INDEX_PREFIX index
+ENV INDEX_PREFIX wikidata-full
 # Need the shell to get the INDEX_PREFIX envirionment variable
 ENTRYPOINT ["/bin/sh", "-c", "exec ServerMain -i \"/index/${INDEX_PREFIX}\" -p 7001 \"$@\"", "--"]
-CMD ["-t", "-a", "-P"]
+CMD ["-a", "-j 8"]
 
 # docker build -t qlever-<name> .
 # # When running with user namespaces you may need to make the index folder accessible

e2e/scientists_queries.yaml

@@ -249,7 +249,7 @@ queries:
           - contains_row: ["<Aaron_Antonovsky>","<Helen_Antonovsky>"]
           - contains_row: ["<Abraham_Zelmanov>", ""]
           - contains_row: ["<Abraham_Pais>","<Ida_Nicolaisen>;<Lila_Lee_Pais>"]
-          - contains_row: ["<Aafia_Siddiqui>","<Ammar_al-Baluchi>;<Amjad_Mohammed_Khan>"]
+          - contains_row: ["<Aafia_Siddiqui>","<Amjad_Mohammed_Khan>;<Ammar_al-Baluchi>"]
   - query: giant-int-scientists
     solutions:
       - type: no-text
@@ -338,3 +338,33 @@ queries:
           - contains_row: ["<Albert_Einstein>", "<Nobel_Prize_in_Physics>"]
           - contains_row: ["<Albert_Fert>", "<Wolf_Prize_in_Physics>"]
           - contains_row: ["<Albert_Overhauser>", "<National_Medal_of_Science_for_Physical_Science>"]
+  - query : having-predicate-religion 
+    solutions:
+      - type: no-text
+        sparql: |
+          SELECT ?predicate (COUNT(?predicate) as ?count) WHERE {
+            ?x <is-a> <Astronaut> .
+            ?x ql:has-predicate ?predicate .
+          }
+          GROUP BY ?predicate
+          HAVING (?predicate < <Z) (?predicate = <Religion>)
+        checks:
+          - num_rows: 1
+          - num_cols: 2
+          - selected: ["?predicate", "?count"]
+          - contains_row: ["<Religion>", "5"]
+  - query : pattern-trick-automatic-having 
+    solutions:
+      - type: no-text
+        sparql: |
+          SELECT ?predicate (COUNT(?predicate) as ?count) WHERE {
+            ?x ql:has-predicate ?predicate .
+            FILTER (?predicate = <Gender>)
+          }
+          GROUP BY ?predicate
+          ORDER BY DESC(?count)
+        checks:
+          - num_rows: 1 
+          - num_cols: 2
+          - selected: ["?predicate", "?count"]
+          - contains_row: ["<Gender>", "18589"]

src/TurtleParserMain.cpp

@@ -0,0 +1,21 @@
+// Copyright 2018, University of Freiburg,
+// Chair of Algorithms and Data Structures.
+// Author: Johannes Kalmbach(joka921) <johannes.kalmbach@gmail.com>
+
+#include <array>
+#include <iostream>
+#include <string>
+#include "./parser/TurtleParser.h"
+
+int main(int argc, char** argv) {
+  if (argc != 2) {
+    std::cerr << "Usage: ./TurtleParserMain <turtleInput>";
+    exit(1);
+  }
+  TurtleParser p(argv[1]);
+  std::array<std::string, 3> triple;
+  while (p.getLine(&triple)) {
+    std::cout << triple[0] << " " << triple[1] << " " << triple[2] << '\n';
+  }
+}
+

src/engine/CountAvailablePredicates.cpp

@@ -41,9 +41,9 @@ string CountAvailablePredicates::asString(size_t indent) const {
 size_t CountAvailablePredicates::getResultWidth() const { return 2; }
 
 // _____________________________________________________________________________
-size_t CountAvailablePredicates::resultSortedOn() const {
+vector<size_t> CountAvailablePredicates::resultSortedOn() const {
   // The result is not sorted on any column.
-  return std::numeric_limits<size_t>::max();
+  return {};
 }
 
 // _____________________________________________________________________________
@@ -100,7 +100,7 @@ size_t CountAvailablePredicates::getCostEstimate() {
 // _____________________________________________________________________________
 void CountAvailablePredicates::computeResult(ResultTable* result) const {
   result->_nofColumns = 2;
-  result->_sortedBy = 0;
+  result->_sortedBy = resultSortedOn();
   result->_fixedSizeData = new vector<array<Id, 2>>();
   result->_resultTypes.push_back(ResultTable::ResultType::KB);
   result->_resultTypes.push_back(ResultTable::ResultType::VERBATIM);

src/engine/CountAvailablePredicates.h

@@ -41,32 +41,32 @@ class CountAvailablePredicates : public Operation {
                            std::shared_ptr<QueryExecutionTree> subtree,
                            size_t subjectColumnIndex);
 
-  virtual string asString(size_t indent = 0) const;
+  virtual string asString(size_t indent = 0) const override;
 
-  virtual size_t getResultWidth() const;
+  virtual size_t getResultWidth() const override;
 
-  virtual size_t resultSortedOn() const;
+  virtual vector<size_t> resultSortedOn() const override;
 
   std::unordered_map<string, size_t> getVariableColumns() const;
 
-  virtual void setTextLimit(size_t limit) {
+  virtual void setTextLimit(size_t limit) override {
     if (_subtree != nullptr) {
       _subtree->setTextLimit(limit);
     }
   }
 
-  virtual bool knownEmptyResult() {
+  virtual bool knownEmptyResult() override {
     if (_subtree != nullptr) {
       return _subtree->knownEmptyResult();
     }
     return false;
   }
 
-  virtual float getMultiplicity(size_t col);
+  virtual float getMultiplicity(size_t col) override;
 
-  virtual size_t getSizeEstimate();
+  virtual size_t getSizeEstimate() override;
 
-  virtual size_t getCostEstimate();
+  virtual size_t getCostEstimate() override;
 
   void setVarNames(const std::string& predicateVarName,
                    const std::string& countVarName);
@@ -103,5 +103,5 @@ class CountAvailablePredicates : public Operation {
   std::string _predicateVarName;
   std::string _countVarName;
 
-  virtual void computeResult(ResultTable* result) const;
+  virtual void computeResult(ResultTable* result) const override;
 };

src/engine/Distinct.h

@@ -25,27 +25,35 @@ class Distinct : public Operation {
            std::shared_ptr<QueryExecutionTree> subtree,
            const vector<size_t>& keepIndices);
 
-  virtual string asString(size_t indent = 0) const;
+  virtual string asString(size_t indent = 0) const override;
 
-  virtual size_t resultSortedOn() const { return _subtree->resultSortedOn(); }
+  virtual vector<size_t> resultSortedOn() const override {
+    return _subtree->resultSortedOn();
+  }
 
-  virtual void setTextLimit(size_t limit) { _subtree->setTextLimit(limit); }
+  virtual void setTextLimit(size_t limit) override {
+    _subtree->setTextLimit(limit);
+  }
 
-  virtual size_t getSizeEstimate() { return _subtree->getSizeEstimate(); }
+  virtual size_t getSizeEstimate() override {
+    return _subtree->getSizeEstimate();
+  }
 
-  virtual size_t getCostEstimate() {
+  virtual size_t getCostEstimate() override {
     return getSizeEstimate() + _subtree->getCostEstimate();
   }
 
-  virtual float getMultiplicity(size_t col) {
+  virtual float getMultiplicity(size_t col) override {
     return _subtree->getMultiplicity(col);
   }
 
-  virtual bool knownEmptyResult() { return _subtree->knownEmptyResult(); }
+  virtual bool knownEmptyResult() override {
+    return _subtree->knownEmptyResult();
+  }
 
  private:
   std::shared_ptr<QueryExecutionTree> _subtree;
   vector<size_t> _keepIndices;
 
-  virtual void computeResult(ResultTable* result) const;
+  virtual void computeResult(ResultTable* result) const override;
 };