chore(third_party): add licenses for jieba, lucene++, boost and support mirror download

LICENSE

@@ -373,3 +373,78 @@ Home page: https://roaringbitmap.org/
 License: https://www.apache.org/licenses/LICENSE-2.0
 
 --------------------------------------------------------------------------------
+
+This product includes code from boost.
+
+* Boost source code in third_party/boost/ directory
+
+Copyright: 2003-2023 The Boost Authors
+Home page: https://www.boost.org/
+License: https://www.boost.org/LICENSE_1_0.txt
+
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------
+
+This product includes code from LucenePlusPlus.
+
+* LucenePlusPlus utility in src/paimon/global_index/lucene/ directory
+
+Copyright: 2009-2014 Alan Wright.
+Home page: https://github.com/luceneplusplus/LucenePlusPlus
+License: https://www.apache.org/licenses/LICENSE-2.0
+
+--------------------------------------------------------------------------------
+
+This product includes code from cppjieba.
+
+* cppjieba utility in src/paimon/global_index/lucene/ directory
+
+Copyright: 2013
+Home page: https://github.com/yanyiwu/cppjieba
+License: https://opensource.org/licenses/MIT
+
+The MIT License (MIT)
+
+Copyright (c) 2013
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+--------------------------------------------------------------------------------

NOTICE

@@ -28,3 +28,12 @@ Copyright (C) 2012-2023 Yann Collet
 
 This product includes software from CRoaring project (Apache 2.0)
 Copyright 2016-2022 The CRoaring authors
+
+This product includes software from boost project (BSL 1.0)
+Copyright 2003-2023 The Boost authors
+
+This product includes software from LucenePlusPlus project (Apache 2.0)
+Copyright 2009-2014 Alan Wright.
+
+This product includes software from cppjieba project (MIT)
+Copyright 2013

cmake_modules/ThirdpartyToolchain.cmake

@@ -124,6 +124,30 @@ else()
     endif()
 endif()
 
+if(DEFINED ENV{PAIMON_LIMONP_URL})
+    set(LIMONP_SOURCE_URL "$ENV{PAIMON_LIMONP_URL}")
+else()
+    if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_LIMONP_PKG_NAME}")
+        set_urls(LIMONP_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_LIMONP_PKG_NAME}")
+    else()
+        set_urls(LIMONP_SOURCE_URL
+                 "${THIRDPARTY_MIRROR_URL}https://github.com/yanyiwu/limonp/archive/refs/tags/${PAIMON_LIMONP_PKG_NAME}"
+        )
+    endif()
+endif()
+
+if(DEFINED ENV{PAIMON_JIEBA_URL})
+    set(JIEBA_SOURCE_URL "$ENV{PAIMON_JIEBA_URL}")
+else()
+    if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_JIEBA_PKG_NAME}")
+        set_urls(JIEBA_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_JIEBA_PKG_NAME}")
+    else()
+        set_urls(JIEBA_SOURCE_URL
+                 "${THIRDPARTY_MIRROR_URL}https://github.com/yanyiwu/cppjieba/archive/refs/tags/${PAIMON_JIEBA_PKG_NAME}"
+        )
+    endif()
+endif()
+
 if(DEFINED ENV{PAIMON_GLOG_URL})
     set(GLOG_SOURCE_URL "$ENV{PAIMON_GLOG_URL}")
 else()
@@ -289,6 +313,10 @@ set(EP_COMMON_CMAKE_ARGS
 
 macro(build_lucene)
     message(STATUS "Building lucene from source")
+
+    get_target_property(LUCENE_ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES)
+    get_filename_component(LUCENE_ZLIB_ROOT "${LUCENE_ZLIB_INCLUDE_DIR}" DIRECTORY)
+
     set(LUCENE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lucene_ep-install")
     set(LUCENE_CMAKE_ARGS
         ${EP_COMMON_CMAKE_ARGS}
@@ -302,6 +330,9 @@ macro(build_lucene)
         "-DBOOST_ROOT=${BOOST_INSTALL}"
         "-DBoost_CHRONO_FOUND=TRUE"
         "-DBoost_THREAD_FOUND=TRUE"
+        "-DZLIB_INCLUDE_DIRS=${ZLIB_INCLUDE_DIR}"
+        "-DZLIB_LIBRARY_RELEASE=${ZLIB_LIBRARIES}"
+        "-DZLIB_ROOT=${LUCENE_ZLIB_ROOT}"
         "-DCMAKE_INSTALL_PREFIX=${LUCENE_PREFIX}")
 
     set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.a")
@@ -311,7 +342,8 @@ macro(build_lucene)
                         URL_HASH "SHA256=${PAIMON_LUCENE_BUILD_SHA256_CHECKSUM}"
                         CMAKE_ARGS ${LUCENE_CMAKE_ARGS}
                         BUILD_BYPRODUCTS ${LUCENE_LIB}
-                        DEPENDS boost_date_time
+                        DEPENDS zlib
+                                boost_date_time
                                 boost_filesystem
                                 boost_regex
                                 boost_thread
@@ -331,7 +363,8 @@ macro(build_lucene)
                                      "${LUCENE_INCLUDE_DIR}")
 
     target_link_libraries(lucene
-                          INTERFACE boost_date_time
+                          INTERFACE zlib
+                                    boost_date_time
                                     boost_filesystem
                                     boost_regex
                                     boost_thread
@@ -345,6 +378,13 @@ macro(build_lucene)
 endmacro()
 
 macro(build_jieba)
+    message(STATUS "Building limonp from source")
+    set(LIMONP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/limonp_ep-prefix")
+    externalproject_add(limonp_ep
+                        URL ${LIMONP_SOURCE_URL}
+                        URL_HASH "SHA256=${PAIMON_LIMONP_BUILD_SHA256_CHECKSUM}"
+                        INSTALL_COMMAND "")
+
     message(STATUS "Building jieba from source")
     set(JIEBA_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jieba_ep-prefix")
     set(JIEBA_INSTALL "${CMAKE_CURRENT_BINARY_DIR}/jieba_ep-install")
@@ -360,24 +400,22 @@ macro(build_jieba)
     set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/jieba.diff")
     externalproject_add(jieba_ep
                         ${EP_COMMON_OPTIONS}
-                        GIT_REPOSITORY https://github.com/yanyiwu/cppjieba.git
-                        GIT_TAG ${PAIMON_JIEBA_BUILD_VERSION}
-                        GIT_SHALLOW FALSE
-                        GIT_PROGRESS TRUE
-                        GIT_SUBMODULES_RECURSE TRUE
+                        URL ${JIEBA_SOURCE_URL}
+                        URL_HASH "SHA256=${PAIMON_JIEBA_BUILD_SHA256_CHECKSUM}"
                         CMAKE_ARGS ${JIEBA_CMAKE_ARGS}
                         LOG_PATCH ON
                         PATCH_COMMAND ${CMAKE_COMMAND} -E chdir <SOURCE_DIR> bash -c
                                       "[ -f .patched ] && echo '<SOURCE_DIR> patch already applied, ignore...' || patch -s -N -p1 -i '${PATCH_FILE}' && touch .patched"
                         INSTALL_COMMAND bash -c
-                                        "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/deps/limonp/include/* ${JIEBA_INSTALL}/include/"
+                                        "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${LIMONP_PREFIX}/src/limonp_ep/include/* ${JIEBA_INSTALL}/include/"
     )
 
     # The include directory must exist before it is referenced by a target.
     include_directories(SYSTEM ${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR})
     add_library(jieba INTERFACE IMPORTED)
     target_include_directories(jieba SYSTEM
                                INTERFACE "${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}")
+    add_dependencies(jieba_ep limonp_ep)
     add_dependencies(jieba jieba_ep)
 endmacro()
 
@@ -468,11 +506,8 @@ macro(build_boost)
         ${BOOST_LIBRARY_DIR}/libboost_iostreams.a)
 
     externalproject_add(boost_ep
-                        GIT_REPOSITORY https://github.com/boostorg/boost.git
-                        GIT_TAG boost-${PAIMON_BOOST_BUILD_VERSION}
-                        GIT_SHALLOW FALSE
-                        GIT_PROGRESS TRUE
-                        GIT_SUBMODULES_RECURSE TRUE
+                        URL "${THIRDPARTY_DIR}/boost/${PAIMON_BOOST_PKG_NAME}"
+                        URL_HASH "SHA256=${PAIMON_BOOST_BUILD_SHA256_CHECKSUM}"
                         CONFIGURE_COMMAND ${BOOST_PREFIX}/src/boost_ep/bootstrap.sh
                                           --with-libraries=date_time,filesystem,iostreams,regex,system,thread,chrono,atomic
                         BUILD_IN_SOURCE TRUE

third_party/versions.txt

@@ -71,13 +71,21 @@ PAIMON_JINDOSDK_C_BUILD_VERSION=6.10.2
 PAIMON_JINDOSDK_C_BUILD_SHA256_CHECKSUM=23e61c9815fab1cd88c369445bdbe1eab02cc09bafed3bb5118ecaf5b2fbc518
 PAIMON_JINDOSDK_C_PKG_NAME=jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}.tar.gz
 
-PAIMON_BOOST_BUILD_VERSION=1.66.0
-
 PAIMON_LUCENE_BUILD_VERSION=3.0.9
 PAIMON_LUCENE_BUILD_SHA256_CHECKSUM=4e69e29d5d79a976498ef71eab70c9c88c7014708be4450a9fda7780fe93584e
 PAIMON_LUCENE_PKG_NAME=rel_${PAIMON_LUCENE_BUILD_VERSION}.tar.gz
 
+PAIMON_LIMONP_BUILD_VERSION=1.0.1
+PAIMON_LIMONP_BUILD_SHA256_CHECKSUM=c7b18794f020dbaa1006229b49a39217a463da0cb3586aee83eb7471f4ae71df
+PAIMON_LIMONP_PKG_NAME=v${PAIMON_LIMONP_BUILD_VERSION}.tar.gz
+
 PAIMON_JIEBA_BUILD_VERSION=v5.6.0
+PAIMON_JIEBA_BUILD_SHA256_CHECKSUM=e6e517b778e0f4a99cbed1ee3eaa041616b74bc685e03a6ca08887ad9cedfe49
+PAIMON_JIEBA_PKG_NAME=${PAIMON_JIEBA_BUILD_VERSION}.tar.gz
+
+PAIMON_BOOST_BUILD_VERSION=1_66_0
+PAIMON_BOOST_BUILD_SHA256_CHECKSUM=28e9200637800fbfd1292b2c6876189dba7e8e1c5282c71fac6515e96f7af2b0
+PAIMON_BOOST_PKG_NAME=boost_${PAIMON_BOOST_BUILD_VERSION}.tar.gz
 
 # The first field is the name of the environment variable expected by cmake.
 # This _must_ match what is defined. The second field is the name of the
@@ -99,4 +107,7 @@ DEPENDENCIES=(
   "PAIMON_RAPIDJSON_URL ${PAIMON_RAPIDJSON_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/miloyip/rapidjson/archive/${PAIMON_RAPIDJSON_BUILD_VERSION}.tar.gz"
   "PAIMON_JINDOSDK_C_URL ${PAIMON_JINDOSDK_C_PKG_NAME} https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/${PAIMON_JINDOSDK_C_BUILD_VERSION}/jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}-linux.tar.gz"
   "PAIMON_LUCENE_URL ${PAIMON_LUCENE_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/luceneplusplus/LucenePlusPlus/archive/refs/tags/${PAIMON_LUCENE_PKG_NAME}"
+  "PAIMON_LIMONP_URL ${PAIMON_LIMONP_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/yanyiwu/limonp/archive/refs/tags/${PAIMON_LIMONP_PKG_NAME}"
+  "PAIMON_JIEBA_URL ${PAIMON_JIEBA_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/yanyiwu/cppjieba/archive/refs/tags/${PAIMON_JIEBA_PKG_NAME}"
+  "PAIMON_BOOST_URL ${PAIMON_BOOST_PKG_NAME} ${THIRDPARTY_DIR}/boost/${PAIMON_BOOST_PKG_NAME}"
 )