KEP-1364 Enhance git lfs package scripts and document with an example…

… upgrade
bluzelle · May 3, 2019 · df033c0 · df033c0
1 parent 8952fd9
commit df033c0
Show file tree

Hide file tree

Showing 6 changed files with 177 additions and 42 deletions.
diff --git a/depend/README.md b/depend/README.md
@@ -0,0 +1,79 @@
+swarmDB uses several precompiled libraries for faster CI builds.
+
+##### Set the required library build number in depend/CMakelists:
+```text  
+    ...
+    
+    add_external_project(boost 1.70.0)
+    add_external_project(googletest 1.8.0)
+    add_external_project(jsoncpp 1.8.4)
+    add_external_project(openssl 1.1.1)
+    add_external_project(rocksdb 5.14.3)
+```    
+##### To add the debug version of Boost 1.70.0 using git LFS:
+###### For release use: -DCMAKE_BUILD_TYPE=Release
+```text
+$ cd depend/boost/package
+$ mkdir build-debug && cd build-debug
+$ cmake .. -DCMAKE_BUILD_TYPE=Debug -DPKG_VER=1.70.0 [-DPKG_HASH=<sha256>]
+ ```
+
+ ###### PKG_HASH is optional and if given will be used to validate the downloaded source code archive.   
+ ```text
+-- The CXX compiler identification is GNU 8.3.0
+-- Check for working CXX compiler: /usr/bin/c++
+-- Check for working CXX compiler: /usr/bin/c++ -- works
+-- Detecting CXX compiler ABI info
+-- Detecting CXX compiler ABI info - done
+-- Detecting CXX compile features
+-- Detecting CXX compile features - done
+-- boost -- debug 1.70.0 (chrono,program_options,random,regex,system,thread,log,serialization)
+-- boost -- URL: https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.gz
+-- boost -- URL_HASH: 882b48708d211a5f48e60b0124cf5863c1534cd544ecd0664bb534a4b5d506e9
+-- Configuring done
+-- Generating done
+-- Build files have been written to: /home/username/bluzelle/swarmdb/depend/boost/package/build-debug
+
+$ make
+Scanning dependencies of target boost
+[ 12%] Creating directories for 'boost'
+[ 25%] Performing download step (download, verify and extract) for 'boost'
+-- Downloading...
+   dst='/home/username/bluzelle/swarmdb/depend/boost/package/build-debug/boost/src/boost_1_70_0.tar.gz'
+   timeout='120 seconds'
+-- Using src='https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.gz'
+-- verifying file...
+       file='/home/username/bluzelle/swarmdb/depend/boost/package/build-debug/boost/src/boost_1_70_0.tar.gz'
+-- Downloading... done
+-- extracting...
+     src='/home/username/bluzelle/swarmdb/depend/boost/package/build-debug/boost/src/boost_1_70_0.tar.gz'
+     dst='/home/username/bluzelle/swarmdb/depend/boost/package/build-debug/boost/src/boost'
+-- extracting... [tar xfz]
+-- extracting... [analysis]
+-- extracting... [rename]
+-- extracting... [clean up]
+-- extracting... done
+    ...   
+[ 87%] No install step for 'boost'
+[100%] Completed 'boost'
+[100%] Built target boost
+$
+```
+
+##### Create a package to be added to the repo
+```text
+$ make package
+[100%] Built target boost
+Run CPack packaging tool...
+CPack: Create package using TGZ
+CPack: Install projects
+CPack: - Run preinstall target for: boost
+CPack: - Install project: boost
+CPack: Create package
+CPack: - package: /home/username/bluzelle/swarmdb/depend/boost/package/build-debug/boost-debug-1.70.0-linux.tar.gz generated.
+$ mv boost-debug-1.70.0-linux.tar.gz ..
+$ cd ..
+$ git lfs track boost-debug-1.70.0-linux.tar.gz
+$ git add .gitattributes boost-debug-1.70.0-linux.tar.gz
+$ git commit -m "added debug Boost 1.70.0 for Linux"
+```
diff --git a/depend/boost/package/CMakeLists.txt b/depend/boost/package/CMakeLists.txt
@@ -19,27 +19,40 @@ project(boost CXX)
 include(ExternalProject)
 include(ProcessorCount)
 
+if (NOT DEFINED PKG_VER)
+  message(FATAL_ERROR "usage: -DPKG_VER=x.y.z [-DPKG_HASH=<sha256>]")
+endif()
+
+if(PKG_HASH)
+  set(URL_HASH_OPTION SHA256=${PKG_HASH})
+else()
+  message(WARNING "PKG_HASH not specified! Downloaded package will not be validated.")
+endif()
+
 # default to debug builds
 if("${CMAKE_BUILD_TYPE}" STREQUAL "")
   set(CMAKE_BUILD_TYPE Debug)
 endif()
 
-set(PKG_VER "1.70.0")
 string(TOLOWER ${CMAKE_BUILD_TYPE} PKG_BUILD_TYPE)
-set(URL_HASH "882b48708d211a5f48e60b0124cf5863c1534cd544ecd0664bb534a4b5d506e9")
 set(PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}_${PKG_BUILD_TYPE})
 
 set(BOOST_TARBALL "boost_${PKG_VER}")
 string(REPLACE "." "_" BOOST_TARBALL ${BOOST_TARBALL})
 string(APPEND BOOST_TARBALL ".tar.gz")
 set(BOOST_LIBS "chrono,program_options,random,regex,system,thread,log,serialization")
+set(URL https://dl.bintray.com/boostorg/release/${PKG_VER}/source/${BOOST_TARBALL})
 
 ProcessorCount(NPROC)
 
+message(STATUS "${PROJECT_NAME} -- ${PKG_BUILD_TYPE} ${PKG_VER} (${BOOST_LIBS})")
+message(STATUS "${PROJECT_NAME} -- URL: ${URL}")
+message(STATUS "${PROJECT_NAME} -- URL_HASH: ${PKG_HASH}")
+
 ExternalProject_Add(boost
     PREFIX ${CMAKE_CURRENT_BINARY_DIR}/boost
-    URL https://dl.bintray.com/boostorg/release/${PKG_VER}/source/${BOOST_TARBALL}
-    URL_HASH SHA256=${URL_HASH}
+    URL ${URL}
+    URL_HASH ${URL_HASH_OPTION}
     TIMEOUT 120
     CONFIGURE_COMMAND ${CMAKE_CURRENT_BINARY_DIR}/boost/src/boost/bootstrap.sh --prefix=${PREFIX} --with-libraries=${BOOST_LIBS}
     BUILD_COMMAND ${CMAKE_CURRENT_BINARY_DIR}/boost/src/boost/b2 variant=${PKG_BUILD_TYPE} link=static -j${NPROC} install
@@ -49,15 +62,14 @@ ExternalProject_Add(boost
     )
 
 set(CPACK_GENERATOR TGZ)
-set(CPACK_PACKAGE_VERSION_MAJOR "1")
-set(CPACK_PACKAGE_VERSION_MINOR "70")
-set(CPACK_PACKAGE_VERSION_PATCH "0")
+string(REPLACE "." ";" VERSION_LIST ${PKG_VER})
+list(GET VERSION_LIST 0 CPACK_PACKAGE_VERSION_MAJOR)
+list(GET VERSION_LIST 1 CPACK_PACKAGE_VERSION_MINOR)
+list(GET VERSION_LIST 2 CPACK_PACKAGE_VERSION_PATCH)
 set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0)
 set(CPACK_PACKAGE_NAME ${PROJECT_NAME}-${PKG_BUILD_TYPE})
 string(TOLOWER ${CMAKE_SYSTEM_NAME} CPACK_SYSTEM_NAME)
 
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}_${PKG_BUILD_TYPE}/ DESTINATION ${PKG_BUILD_TYPE}/)
 
 include(CPack)
-
-message(STATUS "${PROJECT_NAME} -- ${CPACK_SYSTEM_NAME} ${PKG_BUILD_TYPE} ${PKG_VER} (${BOOST_LIBS})")
diff --git a/depend/googletest/package/CMakeLists.txt b/depend/googletest/package/CMakeLists.txt
@@ -18,33 +18,45 @@ project(googletest CXX)
 
 include(ExternalProject)
 
+if (NOT DEFINED PKG_VER)
+	message(FATAL_ERROR "usage: -DPKG_VER=x.y.z [-DPKG_HASH=<sha256>]")
+endif()
+
+if(PKG_HASH)
+	set(URL_HASH_OPTION SHA256=${PKG_HASH})
+else()
+	message(WARNING "PKG_HASH not specified! Downloaded package will not be validated.")
+endif()
+
 # default to debug builds
 if("${CMAKE_BUILD_TYPE}" STREQUAL "")
   set(CMAKE_BUILD_TYPE Debug)
 endif()
 
-set(PKG_VER "1.8.0")
 string(TOLOWER ${CMAKE_BUILD_TYPE} PKG_BUILD_TYPE)
-set(URL_HASH "58a6f4277ca2bc8565222b3bbd58a177609e9c488e8a72649359ba51450db7d8")
+set(URL https://github.com/google/googletest/archive/release-${PKG_VER}.tar.gz)
 set(PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}_${PKG_BUILD_TYPE})
 
 message(STATUS "${PROJECT_NAME} -- ${PKG_BUILD_TYPE} ${PKG_VER}")
+message(STATUS "${PROJECT_NAME} -- URL: ${URL}")
+message(STATUS "${PROJECT_NAME} -- URL_HASH: ${PKG_HASH}")
 
 ExternalProject_Add(googletest
     PREFIX "${CMAKE_CURRENT_BINARY_DIR}/googletest"
-    URL https://github.com/google/googletest/archive/release-${PKG_VER}.tar.gz
-    URL_HASH SHA256=${URL_HASH}
+    URL ${URL}
+    URL_HASH ${URL_HASH_OPTION}
     TIMEOUT 30
 	CMAKE_ARGS
 	-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
  	-DCMAKE_INSTALL_PREFIX=${PREFIX}
     DOWNLOAD_NO_PROGRESS true
     )
-  
+
 set(CPACK_GENERATOR TGZ)
-set(CPACK_PACKAGE_VERSION_MAJOR "1")
-set(CPACK_PACKAGE_VERSION_MINOR "8")
-set(CPACK_PACKAGE_VERSION_PATCH "0")
+string(REPLACE "." ";" VERSION_LIST ${PKG_VER})
+list(GET VERSION_LIST 0 CPACK_PACKAGE_VERSION_MAJOR)
+list(GET VERSION_LIST 1 CPACK_PACKAGE_VERSION_MINOR)
+list(GET VERSION_LIST 2 CPACK_PACKAGE_VERSION_PATCH)
 set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0)
 set(CPACK_PACKAGE_NAME ${PROJECT_NAME}-${PKG_BUILD_TYPE})
 string(TOLOWER ${CMAKE_SYSTEM_NAME} CPACK_SYSTEM_NAME)

diff --git a/depend/jsoncpp/package/CMakeLists.txt b/depend/jsoncpp/package/CMakeLists.txt
@@ -18,22 +18,33 @@ project(jsoncpp CXX)
 
 include(ExternalProject)
 
+if (NOT DEFINED PKG_VER)
+	message(FATAL_ERROR "usage: -DPKG_VER=x.y.z [-DPKG_HASH=<sha256>]")
+endif()
+
+if(PKG_HASH)
+	set(URL_HASH_OPTION SHA256=${PKG_HASH})
+else()
+	message(WARNING "PKG_HASH not specified! Downloaded package will not be validated.")
+endif()
+
 # default to debug builds
 if("${CMAKE_BUILD_TYPE}" STREQUAL "")
   set(CMAKE_BUILD_TYPE Debug)
 endif()
 
-set(PKG_VER "1.8.4")
 string(TOLOWER ${CMAKE_BUILD_TYPE} PKG_BUILD_TYPE)
-set(URL_HASH "c49deac9e0933bcb7044f08516861a2d560988540b23de2ac1ad443b219afdb6")
 set(PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}_${PKG_BUILD_TYPE})
+set(URL https://github.com/open-source-parsers/jsoncpp/archive/${PKG_VER}.tar.gz)
 
 message(STATUS "${PROJECT_NAME} -- ${PKG_BUILD_TYPE} ${PKG_VER}")
+message(STATUS "${PROJECT_NAME} -- URL: ${URL}")
+message(STATUS "${PROJECT_NAME} -- URL_HASH: ${PKG_HASH}")
 
 ExternalProject_Add(jsoncpp
     PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jsoncpp"
-    URL https://github.com/open-source-parsers/jsoncpp/archive/${PKG_VER}.tar.gz
-    URL_HASH SHA256=${URL_HASH}
+    URL ${URL}
+	URL_HASH ${URL_HASH_OPTION}
     TIMEOUT 30
     CMAKE_ARGS
 	-DJSONCPP_WITH_PKGCONFIG_SUPPORT=OFF
@@ -45,9 +56,10 @@ ExternalProject_Add(jsoncpp
     )
 
 set(CPACK_GENERATOR TGZ)
-set(CPACK_PACKAGE_VERSION_MAJOR "1")
-set(CPACK_PACKAGE_VERSION_MINOR "8")
-set(CPACK_PACKAGE_VERSION_PATCH "4")
+string(REPLACE "." ";" VERSION_LIST ${PKG_VER})
+list(GET VERSION_LIST 0 CPACK_PACKAGE_VERSION_MAJOR)
+list(GET VERSION_LIST 1 CPACK_PACKAGE_VERSION_MINOR)
+list(GET VERSION_LIST 2 CPACK_PACKAGE_VERSION_PATCH)
 set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0)
 set(CPACK_PACKAGE_NAME ${PROJECT_NAME}-${PKG_BUILD_TYPE})
 string(TOLOWER ${CMAKE_SYSTEM_NAME} CPACK_SYSTEM_NAME)

diff --git a/depend/openssl/package/CMakeLists.txt b/depend/openssl/package/CMakeLists.txt
@@ -19,15 +19,24 @@ project(openssl CXX)
 include(ExternalProject)
 include(ProcessorCount)
 
+if(PKG_HASH)
+    set(URL_HASH_OPTION SHA256=${PKG_HASH})
+else()
+    message(WARNING "PKG_HASH not specified! Downloaded package will not be validated.")
+endif()
+
 # default to debug builds
 if("${CMAKE_BUILD_TYPE}" STREQUAL "")
   set(CMAKE_BUILD_TYPE Debug)
 endif()
 
-set(PKG_VER "1.1.1")
 string(TOLOWER ${CMAKE_BUILD_TYPE} PKG_BUILD_TYPE)
-set(URL_HASH "2836875a0f89c03d0fdf483941512613a50cfb421d6fd94b9f41d7279d586a3d")
 set(PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}_${PKG_BUILD_TYPE})
+set(URL https://www.openssl.org/source/openssl-${PKG_VER}.tar.gz)
+
+message(STATUS "${PROJECT_NAME} -- ${PKG_BUILD_TYPE} ${PKG_VER}")
+message(STATUS "${PROJECT_NAME} -- URL: ${URL}")
+message(STATUS "${PROJECT_NAME} -- URL_HASH: ${PKG_HASH}")
 
 # platform detection
 if (APPLE)
@@ -40,8 +49,8 @@ ProcessorCount(NPROC)
 
 ExternalProject_Add(openssl
     PREFIX "${CMAKE_CURRENT_BINARY_DIR}/openssl"
-    URL https://www.openssl.org/source/openssl-${PKG_VER}.tar.gz
-    URL_HASH SHA256=${URL_HASH}
+    URL ${URL}
+    URL_HASH ${URL_HASH_OPTION}
     TIMEOUT 30
     DOWNLOAD_NO_PROGRESS true
 	CONFIGURE_COMMAND ${CMAKE_CURRENT_BINARY_DIR}/openssl/src/openssl/Configure ${OPENSSL_BUILD_PLATFORM} --prefix=${PREFIX} --${PKG_BUILD_TYPE}
@@ -50,15 +59,14 @@ ExternalProject_Add(openssl
     )
 
 set(CPACK_GENERATOR TGZ)
-set(CPACK_PACKAGE_VERSION_MAJOR "1")
-set(CPACK_PACKAGE_VERSION_MINOR "1")
-set(CPACK_PACKAGE_VERSION_PATCH "1")
+string(REPLACE "." ";" VERSION_LIST ${PKG_VER})
+list(GET VERSION_LIST 0 CPACK_PACKAGE_VERSION_MAJOR)
+list(GET VERSION_LIST 1 CPACK_PACKAGE_VERSION_MINOR)
+list(GET VERSION_LIST 2 CPACK_PACKAGE_VERSION_PATCH)
 set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0)
 set(CPACK_PACKAGE_NAME ${PROJECT_NAME}-${PKG_BUILD_TYPE})
 string(TOLOWER ${CMAKE_SYSTEM_NAME} CPACK_SYSTEM_NAME)
 
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}_${PKG_BUILD_TYPE}/ DESTINATION ${PKG_BUILD_TYPE}/)
 
 include(CPack)
-
-message(STATUS "${PROJECT_NAME} -- ${CPACK_SYSTEM_NAME} ${PKG_BUILD_TYPE} ${PKG_VER}")
diff --git a/depend/rocksdb/package/CMakeLists.txt b/depend/rocksdb/package/CMakeLists.txt
@@ -19,6 +19,16 @@ project(rocksdb CXX)
 include(ExternalProject)
 include(ProcessorCount)
 
+if (NOT DEFINED PKG_VER)
+  message(FATAL_ERROR "usage: -DPKG_VER=x.y.z [-DPKG_HASH=<sha256>]")
+endif()
+
+if(PKG_HASH)
+  set(URL_HASH_OPTION SHA256=${PKG_HASH})
+else()
+  message(WARNING "PKG_HASH not specified! Downloaded package will not be validated.")
+endif()
+
 # default to debug builds
 if("${CMAKE_BUILD_TYPE}" STREQUAL "")
   set(CMAKE_BUILD_TYPE Debug)
@@ -31,17 +41,20 @@ else()
   set(DEBUG_LEVEL "0")
 endif()
 
-set(PKG_VER "5.14.3")
 string(TOLOWER ${CMAKE_BUILD_TYPE} PKG_BUILD_TYPE)
-set(URL_HASH "c7019a645fc23df0adfe97ef08e793a36149bff2f57ef3b6174cbb0c8c9867b1")
 set(PREFIX ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}_${PKG_BUILD_TYPE})
+set(URL https://github.com/facebook/rocksdb/archive/v${PKG_VER}.tar.gz)
 
 ProcessorCount(NPROC)
 
+message(STATUS "${PROJECT_NAME} -- ${PKG_BUILD_TYPE} ${PKG_VER}")
+message(STATUS "${PROJECT_NAME} -- URL: ${URL}")
+message(STATUS "${PROJECT_NAME} -- URL_HASH: ${PKG_HASH}")
+
 ExternalProject_Add(rocksdb
     PREFIX "${CMAKE_CURRENT_BINARY_DIR}/rocksdb"
-    URL https://github.com/facebook/rocksdb/archive/v${PKG_VER}.tar.gz
-    URL_HASH SHA256=${URL_HASH}
+    URL ${URL}
+    URL_HASH ${URL_HASH_OPTION}
     TIMEOUT 30
     CONFIGURE_COMMAND ""
     BUILD_COMMAND PORTABLE=1 make -e DISABLE_JEMALLOC=1 DEBUG_LEVEL=${DEBUG_LEVEL} static_lib -j${NPROC}
@@ -51,15 +64,14 @@ ExternalProject_Add(rocksdb
     )
 
 set(CPACK_GENERATOR TGZ)
-set(CPACK_PACKAGE_VERSION_MAJOR "5")
-set(CPACK_PACKAGE_VERSION_MINOR "14")
-set(CPACK_PACKAGE_VERSION_PATCH "3")
+string(REPLACE "." ";" VERSION_LIST ${PKG_VER})
+list(GET VERSION_LIST 0 CPACK_PACKAGE_VERSION_MAJOR)
+list(GET VERSION_LIST 1 CPACK_PACKAGE_VERSION_MINOR)
+list(GET VERSION_LIST 2 CPACK_PACKAGE_VERSION_PATCH)
 set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY 0)
 set(CPACK_PACKAGE_NAME ${PROJECT_NAME}-${PKG_BUILD_TYPE})
 string(TOLOWER ${CMAKE_SYSTEM_NAME} CPACK_SYSTEM_NAME)
 
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}_${PKG_BUILD_TYPE}/ DESTINATION ${PKG_BUILD_TYPE}/)
 
 include(CPack)
-
-message(STATUS "${PROJECT_NAME} -- ${CPACK_SYSTEM_NAME} ${PKG_BUILD_TYPE} ${PKG_VER}")