pisa-engine · elshize · Oct 22, 2019 · Oct 22, 2019 · Oct 23, 2019 · Oct 25, 2019
diff --git a/.clang-format b/.clang-format
@@ -34,7 +34,7 @@ PenaltyBreakFirstLessLess: 20
 PenaltyBreakString: 1000
 PenaltyExcessCharacter: 1000000
 PenaltyReturnTypeOnItsOwnLine: 200
-PointerAlignment: Right
+PointerAlignment: Left
 SpaceAfterControlStatementKeyword: true
 SpaceBeforeAssignmentOperators: true
 SpaceInEmptyParentheses: false

diff --git a/.clang-tidy b/.clang-tidy
@@ -0,0 +1,17 @@
+HeaderFilterRegex: '.*include/pisa.*\.hpp'
+Checks: |
+    *,
+    -clang-diagnostic-c++17-extensions,
+    -llvm-header-guard,
+    -cppcoreguidelines-pro-type-reinterpret-cast,
+    -google-runtime-references,
+    -fuchsia-*,
+    -google-readability-namespace-comments,
+    -llvm-namespace-comment,
+    -clang-diagnostic-error,
+    -cppcoreguidelines-pro-bounds-pointer-arithmetic,
+    -cppcoreguidelines-avoid-magic-numbers,
+    -cppcoreguidelines-pro-bounds-array-to-pointer-decay,
+    -modernize-use-trailing-return-type,
+    -misc-non-private-member-variables-in-classes,
+    -readability-magic-numbers
diff --git a/.gitignore b/.gitignore
@@ -25,3 +25,4 @@ docs/_build/
 node_modules
 
 .clangd/
+compile_commands.json
diff --git a/.gitmodules b/.gitmodules
@@ -67,6 +67,21 @@
 [submodule "external/wapopp"]
 	path = external/wapopp
 	url = https://github.com/pisa-engine/wapopp.git
+[submodule "external/optional"]
+	path = external/optional
+	url = https://github.com/TartanLlama/optional.git
+[submodule "external/expected"]
+	path = external/expected
+	url = https://github.com/TartanLlama/expected.git
+[submodule "external/yaml-cpp"]
+	path = external/yaml-cpp
+	url = https://github.com/jbeder/yaml-cpp.git
 [submodule "external/rapidcheck"]
 	path = external/rapidcheck
 	url = https://github.com/emil-e/rapidcheck.git
+[submodule "external/json"]
+	path = external/json
+	url = https://github.com/nlohmann/json.git
+[submodule "external/cereal"]
+	path = external/cereal
+	url = https://github.com/USCiLab/cereal.git
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -9,6 +9,9 @@ set(CMAKE_CXX_EXTENSIONS OFF)
 option(PISA_BUILD_TOOLS "Build command line tools." ON)
 option(PISA_ENABLE_TESTING "Enable testing of the library." ON)
 option(PISA_ENABLE_BENCHMARKING "Enable benchmarking of the library." ON)
+option(PISA_COMPILE_TOOLS "Compile CLI tools." ON)
+option(FORCE_COLORED_OUTPUT "Always produce ANSI-colored output (GNU/Clang only)." ON)
+option(PISA_LIBCXX "Use libc++ standard library." OFF)
 
 configure_file(
   ${PISA_SOURCE_DIR}/include/pisa/pisa_config.hpp.in
@@ -30,7 +33,7 @@ ExternalProject_Add(gumbo-external
     SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/gumbo-parser
     BINARY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/external/gumbo-parser
     CONFIGURE_COMMAND ./autogen.sh && ./configure --prefix=${CMAKE_BINARY_DIR}/gumbo-parser
-	BUILD_BYPRODUCTS ${CMAKE_BINARY_DIR}/gumbo-parser/lib/libgumbo.a
+    BUILD_BYPRODUCTS ${CMAKE_BINARY_DIR}/gumbo-parser/lib/libgumbo.a
     BUILD_COMMAND ${MAKE})
 add_library(gumbo::gumbo STATIC IMPORTED)
 set_property(TARGET gumbo::gumbo APPEND PROPERTY INTERFACE_INCLUDE_DIRECTORIES
@@ -47,20 +50,38 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/external/CMake-codecov/cmake"
 find_package(codecov)
 list(APPEND LCOV_REMOVE_PATTERNS "'${PROJECT_SOURCE_DIR}/external/*'")
 
-
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-strict-aliasing")
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DGSL_UNENFORCED_ON_CONTRACT_VIOLATION -flto")
 if (UNIX)
    # For hardware popcount and other special instructions
-   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
+   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native -Wno-odr")
 
    # Extensive warnings
-   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wno-missing-braces")
+   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wno-missing-braces")
+   #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wno-missing-braces")
 
    if (USE_SANITIZERS)
      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fno-omit-frame-pointer")
    endif ()
 
-   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb") # Add debug info anyway
+   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ggdb -gdwarf") # Add debug info anyway
+
+   #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
+   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer -Wfatal-errors")
+
+   if (${FORCE_COLORED_OUTPUT})
+       if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
+          set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fdiagnostics-color=always")
+       elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
+          set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fcolor-diagnostics")
+       endif ()
+   endif ()
+
+endif()
 
+if (PISA_LIBCXX)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libc++")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stdlib=libc++ -lc++abi")
 endif()
 
 find_package(OpenMP)
@@ -100,16 +121,21 @@ target_link_libraries(pisa PUBLIC # TODO(michal): are there any of these we can
     spdlog
     fmt::fmt
     range-v3
+    optional
+    yaml-cpp
+    nlohmann_json
 )
 target_include_directories(pisa PUBLIC external)
 
-if (PISA_BUILD_TOOLS)
+if (PISA_COMPILE_TOOLS)
+    add_subdirectory(v1)
     add_subdirectory(tools)
 endif()
 
 if (PISA_ENABLE_TESTING AND BUILD_TESTING)
     enable_testing()
-    add_subdirectory(test)
+    #add_subdirectory(test)
+    add_subdirectory(test/v1)
 endif()
 
 if (PISA_ENABLE_BENCHMARKING)

diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt
@@ -106,8 +106,9 @@ set(TRECPP_BUILD_TOOL OFF CACHE BOOL "skip trecpp testing")
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/trecpp)
 
 # Add trecpp
+set(JSON_MultipleHeaders ON CACHE BOOL "")
 set(WAPOPP_ENABLE_TESTING OFF CACHE BOOL "skip wapopp testing")
-add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/wapopp)
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/wapopp EXCLUDE_FROM_ALL)
 
 # Add fmt
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/fmt)
@@ -118,6 +119,22 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/spdlog)
 # Add range-v3
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/range-v3)
 
+# Add tl::optional
+set(OPTIONAL_ENABLE_TESTS OFF CACHE BOOL "skip tl::optional testing")
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/optional EXCLUDE_FROM_ALL)
+
+# Add yaml-cpp
+set(YAML_CPP_BUILD_TOOLS OFF CACHE BOOL "skip building YAML tools")
+set(YAML_CPP_BUILD_TESTS OFF CACHE BOOL "skip building YAML tests")
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/yaml-cpp EXCLUDE_FROM_ALL)
+
 # Add RapidCheck
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/rapidcheck)
 target_compile_options(rapidcheck PRIVATE -Wno-error=all)
+
+# Add json
+# TODO(michal): I had to comment this out because `wapocpp` already adds this target.
+#               How should we deal with it?
+#set(JSON_MultipleHeaders ON CACHE BOOL "")
+#set(JSON_BuildTests OFF CACHE BOOL "skip building JSON tests")
+#add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/json)
diff --git a/external/cereal b/external/cereal
diff --git a/external/expected b/external/expected
diff --git a/external/json b/external/json
diff --git a/external/optional b/external/optional
diff --git a/external/yaml-cpp b/external/yaml-cpp
diff --git a/include/pisa/codec/integer_codes.hpp b/include/pisa/codec/integer_codes.hpp
@@ -5,7 +5,7 @@
 namespace pisa {
 
 // note: n can be 0
-inline void write_gamma(bit_vector_builder &bvb, uint64_t n)
+inline void write_gamma(bit_vector_builder& bvb, uint64_t n)
 {
     uint64_t nn = n + 1;
     uint64_t l = broadword::msb(nn);
@@ -14,22 +14,27 @@ inline void write_gamma(bit_vector_builder &bvb, uint64_t n)
     bvb.append_bits(nn ^ hb, l);
 }
 
-inline void write_gamma_nonzero(bit_vector_builder &bvb, uint64_t n)
+inline void write_gamma_nonzero(bit_vector_builder& bvb, uint64_t n)
 {
     assert(n > 0);
     write_gamma(bvb, n - 1);
 }
 
-inline uint64_t read_gamma(bit_vector::enumerator &it)
+template <typename BitVectorEnumerator>
+inline uint64_t read_gamma(BitVectorEnumerator& it)
 {
     uint64_t l = it.skip_zeros();
     assert(l < 64);
     return (it.take(l) | (uint64_t(1) << l)) - 1;
 }
 
-inline uint64_t read_gamma_nonzero(bit_vector::enumerator &it) { return read_gamma(it) + 1; }
+template <typename BitVectorEnumerator>
+inline uint64_t read_gamma_nonzero(BitVectorEnumerator& it)
+{
+    return read_gamma(it) + 1;
+}
 
-inline void write_delta(bit_vector_builder &bvb, uint64_t n)
+inline void write_delta(bit_vector_builder& bvb, uint64_t n)
 {
     uint64_t nn = n + 1;
     uint64_t l = broadword::msb(nn);
@@ -38,9 +43,11 @@ inline void write_delta(bit_vector_builder &bvb, uint64_t n)
     bvb.append_bits(nn ^ hb, l);
 }
 
-inline uint64_t read_delta(bit_vector::enumerator &it)
+template <typename BitVectorEnumerator>
+inline uint64_t read_delta(BitVectorEnumerator& it)
 {
     uint64_t l = read_gamma(it);
     return (it.take(l) | (uint64_t(1) << l)) - 1;
 }
+
 } // namespace pisa
diff --git a/include/pisa/codec/simdbp.hpp b/include/pisa/codec/simdbp.hpp
@@ -1,8 +1,8 @@
 #pragma once
 
-#include <vector>
-#include "util/util.hpp"
 #include "codec/block_codecs.hpp"
+#include "util/util.hpp"
+#include <vector>
 
 extern "C" {
 #include "simdcomp/include/simdbitpacking.h"
@@ -14,7 +14,8 @@ struct simdbp_block {
     static void encode(uint32_t const *in,
                        uint32_t sum_of_values,
                        size_t n,
-                       std::vector<uint8_t> &out) {
+                       std::vector<uint8_t> &out)
+    {
 
         assert(n <= block_size);
         uint32_t *src = const_cast<uint32_t *>(in);
@@ -23,23 +24,22 @@ struct simdbp_block {
             return;
         }
         uint32_t b = maxbits(in);
-        thread_local std::vector<uint8_t> buf(8*n);
-        uint8_t * buf_ptr = buf.data();
+        thread_local std::vector<uint8_t> buf(8 * n);
+        uint8_t *buf_ptr = buf.data();
         *buf_ptr++ = b;
         simdpackwithoutmask(src, (__m128i *)buf_ptr, b);
         out.insert(out.end(), buf.data(), buf.data() + b * sizeof(__m128i) + 1);
     }
-    static uint8_t const *decode(uint8_t const *in,
-                                 uint32_t *out,
-                                 uint32_t sum_of_values,
-                                 size_t n) {
+    static uint8_t const *decode(uint8_t const *in, uint32_t *out, uint32_t sum_of_values, size_t n)
+    {
         assert(n <= block_size);
         if (PISA_UNLIKELY(n < block_size)) {
             return interpolative_block::decode(in, out, sum_of_values, n);
         }
         uint32_t b = *in++;
         simdunpack((const __m128i *)in, out, b);
-        return in +  b * sizeof(__m128i);
+        return in + b * sizeof(__m128i);
     }
 };
-} // namespace pisa
+
+} // namespace pisa
diff --git a/include/pisa/cursor/block_max_scored_cursor.hpp b/include/pisa/cursor/block_max_scored_cursor.hpp
@@ -1,9 +1,11 @@
 #pragma once
 
+#include <vector>
+
+#include "query/queries.hpp"
+#include "scorer/bm25.hpp"
 #include "scorer/index_scorer.hpp"
 #include "wand_data.hpp"
-#include "query/queries.hpp"
-#include <vector>
 
 namespace pisa {
 
@@ -20,9 +22,9 @@ struct block_max_scored_cursor {
 };
 
 template <typename Index, typename WandType, typename Scorer>
-[[nodiscard]] auto make_block_max_scored_cursors(Index const &index,
-                                                 WandType const &wdata,
-                                                 Scorer const &scorer,
+[[nodiscard]] auto make_block_max_scored_cursors(Index const& index,
+                                                 WandType const& wdata,
+                                                 Scorer const& scorer,
                                                  Query query)
 {
     auto terms = query.terms;
@@ -34,7 +36,7 @@ template <typename Index, typename WandType, typename Scorer>
         query_term_freqs.begin(),
         query_term_freqs.end(),
         std::back_inserter(cursors),
-        [&](auto &&term) {
+        [&](auto&& term) {
             auto list = index[term.first];
             auto w_enum = wdata.getenum(term.first);
             float q_weight = term.second;
@@ -45,4 +47,4 @@ template <typename Index, typename WandType, typename Scorer>
     return cursors;
 }
 
-} // namespace pisa
+} // namespace pisa
diff --git a/include/pisa/cursor/max_scored_cursor.hpp b/include/pisa/cursor/max_scored_cursor.hpp
@@ -1,9 +1,11 @@
 #pragma once
 
+#include <vector>
+
+#include "query/queries.hpp"
+#include "scorer/bm25.hpp"
 #include "scorer/index_scorer.hpp"
 #include "wand_data.hpp"
-#include "query/queries.hpp"
-#include <vector>
 
 namespace pisa {
 
@@ -17,9 +19,9 @@ struct max_scored_cursor {
 };
 
 template <typename Index, typename WandType, typename Scorer>
-[[nodiscard]] auto make_max_scored_cursors(Index const &index,
-                                           WandType const &wdata,
-                                           Scorer const &scorer,
+[[nodiscard]] auto make_max_scored_cursors(Index const& index,
+                                           WandType const& wdata,
+                                           Scorer const& scorer,
                                            Query query)
 {
     auto terms = query.terms;
@@ -30,7 +32,7 @@ template <typename Index, typename WandType, typename Scorer>
     std::transform(query_term_freqs.begin(),
                    query_term_freqs.end(),
                    std::back_inserter(cursors),
-                   [&](auto &&term) {
+                   [&](auto&& term) {
                        auto list = index[term.first];
                        float q_weight = term.second;
                        auto max_weight = q_weight * wdata.max_term_weight(term.first);