F.run on ubuntu18.04

.github/workflows/docker.yml

@@ -13,6 +13,11 @@ jobs:
     # cross-platform coverage.
     # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        dockerfile: [ Dockerfile, Dockerfiles/Dockerfile.Ubuntu18.04, Dockerfiles/Dockerfile.Ubuntu20.04]
+        os: [ ubuntu-latest ]
 
     steps:
       - uses: actions/checkout@v2
@@ -32,6 +37,7 @@ jobs:
         run: git submodule update --init --recursive
 
       - name: Docker build
-        run: sudo docker build -t qlever ${{github.workspace}}
+        run: sudo docker build -t qlever ${{github.workspace}} -f ${{github.workspace}}/${{matrix.dockerfile}}
       - name: E2E in Docker
         run:  sudo docker run -i --rm -v "${{github.workspace}}/e2e_data:/app/e2e_data/" --entrypoint e2e/e2e.sh qlever
+        if: ${{matrix.dockerfile == 'Dockerfile'}}

CMakeLists.txt

@@ -62,12 +62,24 @@ include_directories(${ICU_INCLUDE_DIR})
 ###################################
 # JEMALLOC
 ###################################
-find_package(PkgConfig REQUIRED)
-pkg_check_modules (JEMALLOC jemalloc)
-
-pkg_search_module(JEMALLOC REQUIRED jemalloc)
-include_directories(${JEMALLOC_INCLUDE_DIRS})
-link_libraries(${JEMALLOC_LIBRARIES})
+if (${JEMALLOC_MANUALLY_INSTALLED})
+  link_libraries(jemalloc)
+else()
+  find_package(PkgConfig)
+  pkg_check_modules (JEMALLOC jemalloc)
+
+  pkg_search_module(JEMALLOC jemalloc)
+  if (${JEMALLOC_FOUND})
+    include_directories(${JEMALLOC_INCLUDE_DIRS})
+    link_libraries(${JEMALLOC_LIBRARIES})
+  else ()
+    message(FATAL_ERROR "Jemalloc is required, but could not be found via
+    pkg-config. If you are sure that you have installed jemalloc on your system
+    (e.g. via `apt install libjemalloc-dev` on Ubuntu, you might try rerunning
+    cmake with `-DJEMALLOC_MANUALLY_INSTALLED=True`). This is currently necessary
+    e.g. on Ubuntu 18.04, where pkg-config does not find jemalloc.")
+  endif()
+endif()
 
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/lib)

Dockerfiles/Dockerfile.Ubuntu18.04

@@ -0,0 +1,65 @@
+FROM ubuntu:18.04 as base
+LABEL maintainer="Johannes Kalmbach <kalmbacj@informatik.uni-freiburg.de>"
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+ENV LC_CTYPE C.UTF-8
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update
+RUN apt install -y software-properties-common
+RUN add-apt-repository -y ppa:mhier/libboost-latest
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test
+RUN apt-get update
+
+FROM base as builder
+# Install the kitware repository for a recent cmake, the version on Ubuntu 18.04 is too old
+RUN apt install -y gpg wget
+RUN wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
+RUN echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ bionic main' | tee /etc/apt/sources.list.d/kitware.list >/dev/null
+RUN apt-get update
+
+RUN apt-get install -y build-essential cmake libicu-dev tzdata pkg-config uuid-runtime uuid-dev git
+RUN apt install -y gcc-11 g++-11
+RUN apt install -y libjemalloc-dev ninja-build libzstd-dev
+RUN apt install -y libboost1.74-dev
+
+COPY . /app/
+
+WORKDIR /app/
+ENV DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /app/build/
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DJEMALLOC_MANUALLY_INSTALLED=True -DCMAKE_CXX_COMPILER="g++-11" -DLOGLEVEL=DEBUG -DUSE_PARALLEL=true -GNinja .. && ninja
+RUN make test
+
+FROM base as runtime
+WORKDIR /app
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y wget python3-yaml unzip curl bzip2 pkg-config libicu-dev python3-icu libgomp1 uuid-runtime
+RUN apt install -y lbzip2 libjemalloc-dev libzstd-dev
+
+ARG UID=1000
+RUN groupadd -r qlever && useradd --no-log-init -r -u $UID -g qlever qlever && chown qlever:qlever /app
+USER qlever
+ENV PATH=/app/:$PATH
+
+COPY --from=builder /app/build/*Main /app/src/web/* /app/
+COPY --from=builder /app/e2e/* /app/e2e/
+ENV PATH=/app/:$PATH
+
+USER qlever
+EXPOSE 7001
+VOLUME ["/input", "/index"]
+
+ENV INDEX_PREFIX index
+ENV MEMORY_FOR_QUERIES 70
+ENV CACHE_MAX_SIZE_GB 30
+ENV CACHE_MAX_SIZE_GB_SINGLE_ENTRY 5
+ENV CACHE_MAX_NUM_ENTRIES 1000
+# Need the shell to get the INDEX_PREFIX envirionment variable
+ENTRYPOINT ["/bin/sh", "-c", "exec ServerMain -i \"/index/${INDEX_PREFIX}\" -j 8 -m ${MEMORY_FOR_QUERIES} -c ${CACHE_MAX_SIZE_GB} -e ${CACHE_MAX_SIZE_GB_SINGLE_ENTRY} -k ${CACHE_MAX_NUM_ENTRIES} -p 7001 \"$@\"", "--"]
+
+# Build image:  docker build -t qlever.master .
+
+# Build index:  DB=wikidata; docker run -it --rm -v "$(pwd)":/index --entrypoint bash --name qlever.$DB-index qlever.master -c "IndexBuilderMain -f /index/$DB.nt -i /index/$DB -s /index/$DB.settings.json | tee /index/$DB.index-log.txt"; rm -f $DB/*tmp*
+
+# Run engine:   DB=wikidata; PORT=7001; docker rm -f qlever.$DB; docker run -d --restart=unless-stopped -v "$(pwd)":/index -p $PORT:7001 -e INDEX_PREFIX=$DB -e MEMORY_FOR_QUERIES=30 --name qlever.$DB qlever.master; docker logs -f --tail=100 qlever.$DB

Dockerfiles/Dockerfile.Ubuntu20.04

@@ -0,0 +1,59 @@
+FROM ubuntu:20.04 as base
+LABEL maintainer="Johannes Kalmbach <kalmbacj@informatik.uni-freiburg.de>"
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+ENV LC_CTYPE C.UTF-8
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update
+RUN apt install -y software-properties-common
+RUN add-apt-repository -y ppa:mhier/libboost-latest
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test
+RUN apt-get update
+
+FROM base as builder
+RUN apt-get install -y build-essential cmake libicu-dev tzdata pkg-config uuid-runtime uuid-dev git
+RUN apt install -y gcc-11 g++-11
+RUN apt install -y libjemalloc-dev ninja-build libzstd-dev
+RUN apt install -y libboost1.74-dev
+
+COPY . /app/
+
+WORKDIR /app/
+ENV DEBIAN_FRONTEND=noninteractive
+
+WORKDIR /app/build/
+RUN cmake -DCMAKE_BUILD_TYPE=Release -DJEMALLOC_MANUALLY_INSTALLED=True -DCMAKE_CXX_COMPILER="g++-11" -DLOGLEVEL=DEBUG -DUSE_PARALLEL=true -GNinja .. && ninja
+RUN make test
+
+FROM base as runtime
+WORKDIR /app
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y wget python3-yaml unzip curl bzip2 pkg-config libicu-dev python3-icu libgomp1 uuid-runtime
+RUN apt install -y lbzip2 libjemalloc-dev libzstd-dev
+
+ARG UID=1000
+RUN groupadd -r qlever && useradd --no-log-init -r -u $UID -g qlever qlever && chown qlever:qlever /app
+USER qlever
+ENV PATH=/app/:$PATH
+
+COPY --from=builder /app/build/*Main /app/src/web/* /app/
+COPY --from=builder /app/e2e/* /app/e2e/
+ENV PATH=/app/:$PATH
+
+USER qlever
+EXPOSE 7001
+VOLUME ["/input", "/index"]
+
+ENV INDEX_PREFIX index
+ENV MEMORY_FOR_QUERIES 70
+ENV CACHE_MAX_SIZE_GB 30
+ENV CACHE_MAX_SIZE_GB_SINGLE_ENTRY 5
+ENV CACHE_MAX_NUM_ENTRIES 1000
+# Need the shell to get the INDEX_PREFIX envirionment variable
+ENTRYPOINT ["/bin/sh", "-c", "exec ServerMain -i \"/index/${INDEX_PREFIX}\" -j 8 -m ${MEMORY_FOR_QUERIES} -c ${CACHE_MAX_SIZE_GB} -e ${CACHE_MAX_SIZE_GB_SINGLE_ENTRY} -k ${CACHE_MAX_NUM_ENTRIES} -p 7001 \"$@\"", "--"]
+
+# Build image:  docker build -t qlever.master .
+
+# Build index:  DB=wikidata; docker run -it --rm -v "$(pwd)":/index --entrypoint bash --name qlever.$DB-index qlever.master -c "IndexBuilderMain -f /index/$DB.nt -i /index/$DB -s /index/$DB.settings.json | tee /index/$DB.index-log.txt"; rm -f $DB/*tmp*
+
+# Run engine:   DB=wikidata; PORT=7001; docker rm -f qlever.$DB; docker run -d --restart=unless-stopped -v "$(pwd)":/index -p $PORT:7001 -e INDEX_PREFIX=$DB -e MEMORY_FOR_QUERIES=30 --name qlever.$DB qlever.master; docker logs -f --tail=100 qlever.$DB

src/util/StringUtils.h

@@ -256,7 +256,9 @@ std::string getLowercaseUtf8(std::string_view s) {
   std::string result;
   icu::StringByteSink<std::string> sink(&result);
   UErrorCode err = U_ZERO_ERROR;
-  icu::CaseMap::utf8ToLower("", 0, s, sink, nullptr, err);
+  icu::CaseMap::utf8ToLower(
+      "", 0, icu::StringPiece{s.data(), static_cast<int32_t>(s.size())}, sink,
+      nullptr, err);
   if (U_FAILURE(err)) {
     throw std::runtime_error(u_errorName(err));
   }