compile: compile rocksdb by cmake (#9)

.travis.yml

@@ -6,20 +6,26 @@ os:
 compiler:
   - gcc
 cache:
+  - ccache
   - apt
 
 addons:
    apt:
       packages: ['zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev']
 env:
-  - TEST_GROUP=platform_dependent # 16-18 minutes
-  - TEST_GROUP=1 # 33-35 minutes
-  - TEST_GROUP=2 # 18-20 minutes
-  - TEST_GROUP=3 # 20-22 minutes
-  - TEST_GROUP=4 # 12-14 minutes
+  - TEST_GROUP=1
+  - TEST_GROUP=2
+  - TEST_GROUP=3
+  - TEST_GROUP=4
+  - TEST_GROUP=5
+  - TEST_GROUP=6
   # Build examples
   - JOB_NAME=examples # 5-7 minutes
 
+install:
+  - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then
+      mkdir cmake-dist && curl -sfSL https://cmake.org/files/v3.8/cmake-3.8.1-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH;
+    fi
 
 before_script:
   # Increase the maximum number of open file descriptors, since some tests use
@@ -28,11 +34,13 @@ before_script:
 
 script:
   - ${CXX} --version
-  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some; fi
-  - if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=full_filter_block_test make -j4 check_some; fi
-  - if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=full_filter_block_test ROCKSDBTESTS_END=write_batch_with_index_test make -j4 check_some; fi
-  - if [ "${TEST_GROUP}" == '3' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=write_batch_with_index_test ROCKSDBTESTS_END=write_prepared_transaction_test make -j4 check_some; fi
-  - if [ "${TEST_GROUP}" == '4' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=write_prepared_transaction_test make -j4 check_some; fi
+  - if [ `command -v ccache` ]; then ccache -C; fi
+  - if [ "${TEST_GROUP}" == '1' ]; then mkdir build; cd build; cmake .. -DWITH_TESTS=ON -DROCKSDBTESTS_END=db/db_log_iter_test.cc; OPT=-DTRAVIS V=1 make -j4; ctest -j4 -V; fi
+  - if [ "${TEST_GROUP}" == '2' ]; then mkdir build; cd build; cmake .. -DWITH_TESTS=ON -DROCKSDBTESTS_START=db/db_log_iter_test.cc -DROCKSDBTESTS_END=db/listener_test.cc; OPT=-DTRAVIS V=1 make -j4; ctest -j4 -V; fi
+  - if [ "${TEST_GROUP}" == '3' ]; then mkdir build; cd build; cmake .. -DWITH_TESTS=ON -DROCKSDBTESTS_START=db/listener_test.cc -DROCKSDBTESTS_END=memtable/skiplist_test.cc; OPT=-DTRAVIS V=1 make -j4; ctest -j4 -V; fi
+  - if [ "${TEST_GROUP}" == '4' ]; then mkdir build; cd build; cmake .. -DWITH_TESTS=ON -DROCKSDBTESTS_START=memtable/skiplist_test.cc -DROCKSDBTESTS_END=util/delete_scheduler_test.cc; OPT=-DTRAVIS V=1 make -j4; ctest -j4 -V; fi
+  - if [ "${TEST_GROUP}" == '5' ]; then mkdir build; cd build; cmake .. -DWITH_TESTS=ON -DROCKSDBTESTS_START=util/delete_scheduler_test.cc -DROCKSDBTESTS_END=utilities/geodb/geodb_test.cc; OPT=-DTRAVIS V=1 make -j4; ctest -j4 -V; fi
+  - if [ "${TEST_GROUP}" == '6' ]; then mkdir build; cd build; cmake .. -DWITH_TESTS=ON -DROCKSDBTESTS_START=utilities/geodb/geodb_test.cc; OPT=-DTRAVIS V=1 make -j4; ctest -j4 -V; fi
   - if [ "${JOB_NAME}" == 'examples' ]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
 
 notifications:

CMakeLists.txt

@@ -41,6 +41,14 @@ endif()
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/modules/")
 
+find_program(CCACHE_FOUND ccache)
+if(CCACHE_FOUND)
+  set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache)
+  set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache)
+  message("use ccache to speed up compilation")
+endif(CCACHE_FOUND)
+
+add_definitions(-DPEGASUS)
 option(WITH_JEMALLOC "build with JeMalloc" OFF)
 if(MSVC)
   include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
@@ -57,23 +65,23 @@ else()
     endif()
   endif()
 
-  option(WITH_SNAPPY "build with SNAPPY" OFF)
+  option(WITH_SNAPPY "build with SNAPPY" ON)
   if(WITH_SNAPPY)
     find_package(snappy REQUIRED)
     add_definitions(-DSNAPPY)
     include_directories(${SNAPPY_INCLUDE_DIR})
     list(APPEND THIRDPARTY_LIBS ${SNAPPY_LIBRARIES})
   endif()
 
-  option(WITH_ZLIB "build with zlib" OFF)
+  option(WITH_ZLIB "build with zlib" ON)
   if(WITH_ZLIB)
     find_package(zlib REQUIRED)
     add_definitions(-DZLIB)
     include_directories(${ZLIB_INCLUDE_DIR})
     list(APPEND THIRDPARTY_LIBS ${ZLIB_LIBRARIES})
   endif()
 
-  option(WITH_BZ2 "build with bzip2" OFF)
+  option(WITH_BZ2 "build with bzip2" ON)
   if(WITH_BZ2)
     find_package(bzip2 REQUIRED)
     add_definitions(-DBZIP2)
@@ -129,7 +137,7 @@ if(NOT WIN32)
   string(STRIP "${ROCKSDB_VERSION_MAJOR}" ROCKSDB_VERSION_MAJOR)
 endif()
 
-option(WITH_MD_LIBRARY "build with MD" ON)
+option(WITH_MD_LIBRARY "build with MD" OFF)
 if(WIN32 AND MSVC)
   if(WITH_MD_LIBRARY)
     set(RUNTIME_LIBRARY "MD")
@@ -592,23 +600,23 @@ endif()
 
 set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
 set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})
-set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
+set(ROCKSDB_IMPORT_LIB ${ROCKSDB_STATIC_LIB})
 if(WIN32)
   set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
   set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 else()
   set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
-  set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+  set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
-  add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
-  target_link_libraries(${ROCKSDB_SHARED_LIB}
-    ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
-  set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
-                        LINKER_LANGUAGE CXX
-                        VERSION ${ROCKSDB_VERSION}
-                        SOVERSION ${ROCKSDB_VERSION_MAJOR}
-                        CXX_STANDARD 11
-                        OUTPUT_NAME "rocksdb")
+#  add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
+#  target_link_libraries(${ROCKSDB_SHARED_LIB}
+#    ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
+#  set_target_properties(${ROCKSDB_SHARED_LIB} PROPERTIES
+#                        LINKER_LANGUAGE CXX
+#                        VERSION ${ROCKSDB_VERSION}
+#                        SOVERSION ${ROCKSDB_VERSION_MAJOR}
+#                        CXX_STANDARD 11
+#                        OUTPUT_NAME "rocksdb")
 endif()
 
 option(WITH_LIBRADOS "Build with librados" OFF)
@@ -682,14 +690,14 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
     INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
   )
 
-  install(
-    TARGETS ${ROCKSDB_SHARED_LIB}
-    EXPORT RocksDBTargets
-    COMPONENT runtime
-    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
-    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
-    INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
-  )
+#  install(
+#    TARGETS ${ROCKSDB_SHARED_LIB}
+#    EXPORT RocksDBTargets
+#    COMPONENT runtime
+#    RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
+#    LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
+#    INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
+#  )
 
   install(
     EXPORT RocksDBTargets
@@ -707,7 +715,7 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
   )
 endif()
 
-option(WITH_TESTS "build with tests" ON)
+option(WITH_TESTS "build with tests" OFF)
 if(WITH_TESTS)
   set(TESTS
         cache/cache_test.cc
@@ -845,6 +853,32 @@ if(WITH_TESTS)
         utilities/ttl/ttl_test.cc
         utilities/write_batch_with_index/write_batch_with_index_test.cc
   )
+
+  if(ROCKSDBTESTS_START)
+    foreach(sourcefile ${TESTS})
+      if(${sourcefile} MATCHES ${ROCKSDBTESTS_START})
+        break()
+      endif()
+      list(REMOVE_ITEM TESTS ${sourcefile})
+    endforeach(sourcefile ${TESTS})
+  endif()
+  if(ROCKSDBTESTS_END)
+    set(index 0)
+    foreach(sourcefile ${TESTS})
+      if(${sourcefile} MATCHES ${ROCKSDBTESTS_END})
+        set(len 0)
+        list(LENGTH TESTS len)
+        while(${index} LESS ${len})
+          list(REMOVE_AT TESTS ${index})
+          list(LENGTH TESTS len)
+        endwhile()
+        break()
+      else()
+        math(EXPR index '${index}+1')
+      endif()
+    endforeach(sourcefile ${TESTS})
+  endif()
+
   if(WITH_LIBRADOS)
     list(APPEND TESTS utilities/env_librados_test.cc)
   endif()

Makefile

@@ -40,19 +40,9 @@ quoted_perl_command = $(subst ','\'',$(perl_command))
 # `make install-shared`, `make static_lib`, `make install-static` or
 # `make install`
 
-CXXFLAGS += -DPEGASUS
-
 # Set the default DEBUG_LEVEL to 1
 DEBUG_LEVEL?=1
 
-ifeq ($(MAKECMDGOALS),static_lib_debug)
-	DEBUG_LEVEL=2
-endif
-
-ifeq ($(MAKECMDGOALS),static_lib_release)
-	DEBUG_LEVEL=0
-endif
-
 ifeq ($(MAKECMDGOALS),dbg)
 	DEBUG_LEVEL=2
 endif
@@ -277,7 +267,7 @@ endif
 default: all
 
 WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
-  -Wno-unused-parameter -Wno-unused-function
+  -Wno-unused-parameter
 
 ifndef DISABLE_WARNING_AS_ERROR
 	WARNING_FLAGS += -Werror
@@ -311,8 +301,8 @@ LDFLAGS += $(LUA_LIB)
 endif
 
 
-CFLAGS += $(WARNING_FLAGS) -I. -I./include -I../src/include $(PLATFORM_CCFLAGS) $(OPT)
-CXXFLAGS += $(WARNING_FLAGS) -I. -I./include -I../src/include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers
+CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
+CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers
 
 LDFLAGS += $(PLATFORM_LDFLAGS)
 
@@ -402,6 +392,7 @@ TESTS = \
 	db_range_del_test \
 	db_sst_test \
 	db_tailing_iter_test \
+	db_universal_compaction_test \
 	db_io_failure_test \
 	db_properties_test \
 	db_table_properties_test \
@@ -503,7 +494,6 @@ TESTS = \
 	repair_test \
 	env_timed_test \
 	write_prepared_transaction_test \
-	db_universal_compaction_test \
 
 PARALLEL_TEST = \
 	backupable_db_test \
@@ -550,12 +540,12 @@ BENCHMARKS = db_bench table_reader_bench cache_bench memtablerep_bench column_aw
 
 # if user didn't config LIBNAME, set the default
 ifeq ($(LIBNAME),)
-## we should only run rocksdb in production with DEBUG_LEVEL 0
-#ifeq ($(DEBUG_LEVEL),0)
+# we should only run rocksdb in production with DEBUG_LEVEL 0
+ifeq ($(DEBUG_LEVEL),0)
         LIBNAME=librocksdb
-#else
-#        LIBNAME=librocksdb_debug
-#endif
+else
+        LIBNAME=librocksdb_debug
+endif
 endif
 LIBRARY = ${LIBNAME}.a
 TOOLS_LIBRARY = ${LIBNAME}_tools.a
@@ -638,8 +628,6 @@ endif  # PLATFORM_SHARED_EXT
 	dbg rocksdbjavastatic rocksdbjava install install-static install-shared uninstall \
 	analyze tools tools_lib
 
-static_lib_debug: $(LIBRARY)
-static_lib_release: $(LIBRARY)
 
 all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)
 

build_tools/build_detect_platform

@@ -269,27 +269,27 @@ EOF
     fi
 
     # Test whether lz4 library is installed
-#    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-#      #include <lz4.h>
-#      #include <lz4hc.h>
-#      int main() {}
-#EOF
-#    if [ "$?" = 0 ]; then
-#        COMMON_FLAGS="$COMMON_FLAGS -DLZ4"
-#        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4"
-#        JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
-#    fi
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <lz4.h>
+      #include <lz4hc.h>
+      int main() {}
+EOF
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DLZ4"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
+    fi
 
     # Test whether zstd library is installed
-#    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-#      #include <zstd.h>
-#      int main() {}
-#EOF
-#    if [ "$?" = 0 ]; then
-#        COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
-#        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
-#        JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
-#    fi
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <zstd.h>
+      int main() {}
+EOF
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
+    fi
 
     # Test whether numa is available
     $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null  <<EOF

tools/sst_dump_tool.cc

@@ -47,7 +47,7 @@ SstFileReader::SstFileReader(const std::string& file_path,
                              bool verify_checksum,
                              bool output_hex)
     :file_name_(file_path), read_num_(0), verify_checksum_(verify_checksum),
-    output_hex_(output_hex), ioptions_(options_),
+    ioptions_(options_),
     internal_comparator_(BytewiseComparator()) {
   fprintf(stdout, "Process %s\n", file_path.c_str());
   init_result_ = GetTableReader(file_name_);

tools/sst_dump_tool_imp.h

@@ -62,7 +62,6 @@ class SstFileReader {
   std::string file_name_;
   uint64_t read_num_;
   bool verify_checksum_;
-  bool output_hex_;
   EnvOptions soptions_;
 
   // options_ and internal_comparator_ will also be used in