diff --git a/LICENSE b/LICENSE index a4f848686..606e7d4de 100644 --- a/LICENSE +++ b/LICENSE @@ -373,3 +373,78 @@ Home page: https://roaringbitmap.org/ License: https://www.apache.org/licenses/LICENSE-2.0 -------------------------------------------------------------------------------- + +This product includes code from boost. + +* Boost source code in third_party/boost/ directory + +Copyright: 2003-2023 The Boost Authors +Home page: https://www.boost.org/ +License: https://www.boost.org/LICENSE_1_0.txt + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- + +This product includes code from LucenePlusPlus. + +* LucenePlusPlus utility in src/paimon/global_index/lucene/ directory + +Copyright: 2009-2014 Alan Wright. +Home page: https://github.com/luceneplusplus/LucenePlusPlus +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from cppjieba. + +* cppjieba utility in src/paimon/global_index/lucene/ directory + +Copyright: 2013 +Home page: https://github.com/yanyiwu/cppjieba +License: https://opensource.org/licenses/MIT + +The MIT License (MIT) + +Copyright (c) 2013 + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +-------------------------------------------------------------------------------- diff --git a/NOTICE b/NOTICE index 242d49778..7f39800cb 100644 --- a/NOTICE +++ b/NOTICE @@ -28,3 +28,12 @@ Copyright (C) 2012-2023 Yann Collet This product includes software from CRoaring project (Apache 2.0) Copyright 2016-2022 The CRoaring authors + +This product includes software from boost project (BSL 1.0) +Copyright 2003-2023 The Boost authors + +This product includes software from LucenePlusPlus project (Apache 2.0) +Copyright 2009-2014 Alan Wright. + +This product includes software from cppjieba project (MIT) +Copyright 2013 diff --git a/cmake_modules/ThirdpartyToolchain.cmake b/cmake_modules/ThirdpartyToolchain.cmake index 120006ef0..975263052 100644 --- a/cmake_modules/ThirdpartyToolchain.cmake +++ b/cmake_modules/ThirdpartyToolchain.cmake @@ -124,6 +124,30 @@ else() endif() endif() +if(DEFINED ENV{PAIMON_LIMONP_URL}) + set(LIMONP_SOURCE_URL "$ENV{PAIMON_LIMONP_URL}") +else() + if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_LIMONP_PKG_NAME}") + set_urls(LIMONP_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_LIMONP_PKG_NAME}") + else() + set_urls(LIMONP_SOURCE_URL + "${THIRDPARTY_MIRROR_URL}https://github.com/yanyiwu/limonp/archive/refs/tags/${PAIMON_LIMONP_PKG_NAME}" + ) + endif() +endif() + +if(DEFINED ENV{PAIMON_JIEBA_URL}) + set(JIEBA_SOURCE_URL "$ENV{PAIMON_JIEBA_URL}") +else() + if(EXISTS "${THIRDPARTY_DIR}/${PAIMON_JIEBA_PKG_NAME}") + set_urls(JIEBA_SOURCE_URL "${THIRDPARTY_DIR}/${PAIMON_JIEBA_PKG_NAME}") + else() + set_urls(JIEBA_SOURCE_URL + "${THIRDPARTY_MIRROR_URL}https://github.com/yanyiwu/cppjieba/archive/refs/tags/${PAIMON_JIEBA_PKG_NAME}" + ) + endif() +endif() + if(DEFINED ENV{PAIMON_GLOG_URL}) set(GLOG_SOURCE_URL "$ENV{PAIMON_GLOG_URL}") else() @@ -289,6 +313,10 @@ set(EP_COMMON_CMAKE_ARGS macro(build_lucene) message(STATUS "Building lucene from source") + + get_target_property(LUCENE_ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) + get_filename_component(LUCENE_ZLIB_ROOT "${LUCENE_ZLIB_INCLUDE_DIR}" DIRECTORY) + set(LUCENE_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lucene_ep-install") set(LUCENE_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} @@ -302,6 +330,9 @@ macro(build_lucene) "-DBOOST_ROOT=${BOOST_INSTALL}" "-DBoost_CHRONO_FOUND=TRUE" "-DBoost_THREAD_FOUND=TRUE" + "-DZLIB_INCLUDE_DIRS=${ZLIB_INCLUDE_DIR}" + "-DZLIB_LIBRARY_RELEASE=${ZLIB_LIBRARIES}" + "-DZLIB_ROOT=${LUCENE_ZLIB_ROOT}" "-DCMAKE_INSTALL_PREFIX=${LUCENE_PREFIX}") set(LUCENE_LIB "${LUCENE_PREFIX}/lib/liblucene++.a") @@ -311,7 +342,8 @@ macro(build_lucene) URL_HASH "SHA256=${PAIMON_LUCENE_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${LUCENE_CMAKE_ARGS} BUILD_BYPRODUCTS ${LUCENE_LIB} - DEPENDS boost_date_time + DEPENDS zlib + boost_date_time boost_filesystem boost_regex boost_thread @@ -331,7 +363,8 @@ macro(build_lucene) "${LUCENE_INCLUDE_DIR}") target_link_libraries(lucene - INTERFACE boost_date_time + INTERFACE zlib + boost_date_time boost_filesystem boost_regex boost_thread @@ -345,6 +378,13 @@ macro(build_lucene) endmacro() macro(build_jieba) + message(STATUS "Building limonp from source") + set(LIMONP_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/limonp_ep-prefix") + externalproject_add(limonp_ep + URL ${LIMONP_SOURCE_URL} + URL_HASH "SHA256=${PAIMON_LIMONP_BUILD_SHA256_CHECKSUM}" + INSTALL_COMMAND "") + message(STATUS "Building jieba from source") set(JIEBA_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/jieba_ep-prefix") set(JIEBA_INSTALL "${CMAKE_CURRENT_BINARY_DIR}/jieba_ep-install") @@ -360,17 +400,14 @@ macro(build_jieba) set(PATCH_FILE "${CMAKE_CURRENT_LIST_DIR}/jieba.diff") externalproject_add(jieba_ep ${EP_COMMON_OPTIONS} - GIT_REPOSITORY https://github.com/yanyiwu/cppjieba.git - GIT_TAG ${PAIMON_JIEBA_BUILD_VERSION} - GIT_SHALLOW FALSE - GIT_PROGRESS TRUE - GIT_SUBMODULES_RECURSE TRUE + URL ${JIEBA_SOURCE_URL} + URL_HASH "SHA256=${PAIMON_JIEBA_BUILD_SHA256_CHECKSUM}" CMAKE_ARGS ${JIEBA_CMAKE_ARGS} LOG_PATCH ON PATCH_COMMAND ${CMAKE_COMMAND} -E chdir bash -c "[ -f .patched ] && echo ' patch already applied, ignore...' || patch -s -N -p1 -i '${PATCH_FILE}' && touch .patched" INSTALL_COMMAND bash -c - "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/deps/limonp/include/* ${JIEBA_INSTALL}/include/" + "cp -r ${JIEBA_PREFIX}/src/jieba_ep/include/* ${JIEBA_INSTALL}/include/ && cp -r ${JIEBA_PREFIX}/src/jieba_ep/dict/* ${JIEBA_INSTALL}/dict/ && cp -r ${LIMONP_PREFIX}/src/limonp_ep/include/* ${JIEBA_INSTALL}/include/" ) # The include directory must exist before it is referenced by a target. @@ -378,6 +415,7 @@ macro(build_jieba) add_library(jieba INTERFACE IMPORTED) target_include_directories(jieba SYSTEM INTERFACE "${JIEBA_INCLUDE_DIR} ${JIEBA_DICT_DIR}") + add_dependencies(jieba_ep limonp_ep) add_dependencies(jieba jieba_ep) endmacro() @@ -468,11 +506,8 @@ macro(build_boost) ${BOOST_LIBRARY_DIR}/libboost_iostreams.a) externalproject_add(boost_ep - GIT_REPOSITORY https://github.com/boostorg/boost.git - GIT_TAG boost-${PAIMON_BOOST_BUILD_VERSION} - GIT_SHALLOW FALSE - GIT_PROGRESS TRUE - GIT_SUBMODULES_RECURSE TRUE + URL "${THIRDPARTY_DIR}/boost/${PAIMON_BOOST_PKG_NAME}" + URL_HASH "SHA256=${PAIMON_BOOST_BUILD_SHA256_CHECKSUM}" CONFIGURE_COMMAND ${BOOST_PREFIX}/src/boost_ep/bootstrap.sh --with-libraries=date_time,filesystem,iostreams,regex,system,thread,chrono,atomic BUILD_IN_SOURCE TRUE diff --git a/third_party/boost/boost_1_66_0.tar.gz b/third_party/boost/boost_1_66_0.tar.gz new file mode 100644 index 000000000..315fbbbd9 --- /dev/null +++ b/third_party/boost/boost_1_66_0.tar.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:28e9200637800fbfd1292b2c6876189dba7e8e1c5282c71fac6515e96f7af2b0 +size 88165095 diff --git a/third_party/versions.txt b/third_party/versions.txt index 76770d571..21e67dee3 100644 --- a/third_party/versions.txt +++ b/third_party/versions.txt @@ -71,13 +71,21 @@ PAIMON_JINDOSDK_C_BUILD_VERSION=6.10.2 PAIMON_JINDOSDK_C_BUILD_SHA256_CHECKSUM=23e61c9815fab1cd88c369445bdbe1eab02cc09bafed3bb5118ecaf5b2fbc518 PAIMON_JINDOSDK_C_PKG_NAME=jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}.tar.gz -PAIMON_BOOST_BUILD_VERSION=1.66.0 - PAIMON_LUCENE_BUILD_VERSION=3.0.9 PAIMON_LUCENE_BUILD_SHA256_CHECKSUM=4e69e29d5d79a976498ef71eab70c9c88c7014708be4450a9fda7780fe93584e PAIMON_LUCENE_PKG_NAME=rel_${PAIMON_LUCENE_BUILD_VERSION}.tar.gz +PAIMON_LIMONP_BUILD_VERSION=1.0.1 +PAIMON_LIMONP_BUILD_SHA256_CHECKSUM=c7b18794f020dbaa1006229b49a39217a463da0cb3586aee83eb7471f4ae71df +PAIMON_LIMONP_PKG_NAME=v${PAIMON_LIMONP_BUILD_VERSION}.tar.gz + PAIMON_JIEBA_BUILD_VERSION=v5.6.0 +PAIMON_JIEBA_BUILD_SHA256_CHECKSUM=e6e517b778e0f4a99cbed1ee3eaa041616b74bc685e03a6ca08887ad9cedfe49 +PAIMON_JIEBA_PKG_NAME=${PAIMON_JIEBA_BUILD_VERSION}.tar.gz + +PAIMON_BOOST_BUILD_VERSION=1_66_0 +PAIMON_BOOST_BUILD_SHA256_CHECKSUM=28e9200637800fbfd1292b2c6876189dba7e8e1c5282c71fac6515e96f7af2b0 +PAIMON_BOOST_PKG_NAME=boost_${PAIMON_BOOST_BUILD_VERSION}.tar.gz # The first field is the name of the environment variable expected by cmake. # This _must_ match what is defined. The second field is the name of the @@ -99,4 +107,7 @@ DEPENDENCIES=( "PAIMON_RAPIDJSON_URL ${PAIMON_RAPIDJSON_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/miloyip/rapidjson/archive/${PAIMON_RAPIDJSON_BUILD_VERSION}.tar.gz" "PAIMON_JINDOSDK_C_URL ${PAIMON_JINDOSDK_C_PKG_NAME} https://jindodata-binary.oss-cn-shanghai.aliyuncs.com/release/${PAIMON_JINDOSDK_C_BUILD_VERSION}/jindosdk-${PAIMON_JINDOSDK_C_BUILD_VERSION}-linux.tar.gz" "PAIMON_LUCENE_URL ${PAIMON_LUCENE_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/luceneplusplus/LucenePlusPlus/archive/refs/tags/${PAIMON_LUCENE_PKG_NAME}" + "PAIMON_LIMONP_URL ${PAIMON_LIMONP_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/yanyiwu/limonp/archive/refs/tags/${PAIMON_LIMONP_PKG_NAME}" + "PAIMON_JIEBA_URL ${PAIMON_JIEBA_PKG_NAME} ${THIRDPARTY_MIRROR_URL}https://github.com/yanyiwu/cppjieba/archive/refs/tags/${PAIMON_JIEBA_PKG_NAME}" + "PAIMON_BOOST_URL ${PAIMON_BOOST_PKG_NAME} ${THIRDPARTY_DIR}/boost/${PAIMON_BOOST_PKG_NAME}" )