From 8e723f5bb98a986e204c2c37299e619954ea413c Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Wed, 6 Nov 2019 06:44:57 +0100 Subject: [PATCH 01/16] Add arrow/0.16.0 recipe --- recipes/arrow/all/CMakeLists.txt | 7 + recipes/arrow/all/conandata.yml | 10 + recipes/arrow/all/conanfile.py | 245 ++++++++++++++++++ .../arrow/all/patches/0001-cmake-16.0.patch | 139 ++++++++++ recipes/arrow/all/patches/0002-jemalloc.patch | 41 +++ recipes/arrow/all/test_package/CMakeLists.txt | 8 + recipes/arrow/all/test_package/conanfile.py | 17 ++ .../arrow/all/test_package/test_package.cpp | 190 ++++++++++++++ recipes/arrow/config.yml | 3 + 9 files changed, 660 insertions(+) create mode 100644 recipes/arrow/all/CMakeLists.txt create mode 100644 recipes/arrow/all/conandata.yml create mode 100644 recipes/arrow/all/conanfile.py create mode 100644 recipes/arrow/all/patches/0001-cmake-16.0.patch create mode 100644 recipes/arrow/all/patches/0002-jemalloc.patch create mode 100644 recipes/arrow/all/test_package/CMakeLists.txt create mode 100644 recipes/arrow/all/test_package/conanfile.py create mode 100644 recipes/arrow/all/test_package/test_package.cpp create mode 100644 recipes/arrow/config.yml diff --git a/recipes/arrow/all/CMakeLists.txt b/recipes/arrow/all/CMakeLists.txt new file mode 100644 index 0000000000000..4d1896ed77c3c --- /dev/null +++ b/recipes/arrow/all/CMakeLists.txt @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 2.8.12) +project(cmake_wrapper) + +include(conanbuildinfo.cmake) +conan_basic_setup() + +add_subdirectory(source_subfolder/cpp) diff --git a/recipes/arrow/all/conandata.yml b/recipes/arrow/all/conandata.yml new file mode 100644 index 0000000000000..3843f7beb5dfd --- /dev/null +++ b/recipes/arrow/all/conandata.yml @@ -0,0 +1,10 @@ +sources: + "0.16.0": + url: "https://github.com/apache/arrow/archive/apache-arrow-0.16.0.tar.gz" + sha256: "d7b3838758a365c8c47d55ab0df1006a70db951c6964440ba354f81f518b8d8d" +patches: + "0.16.0": + - base_path: "source_subfolder" + patch_file: "patches/0001-cmake-16.0.patch" + - base_path: "source_subfolder" + patch_file: "patches/0002-jemalloc.patch" diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py new file mode 100644 index 0000000000000..eafd1108b39d0 --- /dev/null +++ b/recipes/arrow/all/conanfile.py @@ -0,0 +1,245 @@ +from conans import ConanFile, tools, CMake +from conans.errors import ConanInvalidConfiguration +import os + + +class ArrowConan(ConanFile): + name = "arrow" + description = "Apache Arrow is a cross-language development platform for in-memory data" + topics = ("conan", "arrow", "memory") + url = "https://github.com/conan-io/conan-center-index" + homepage = "https://arrow.apache.org/" + license = ("Apache-2.0",) + exports_sources = "CMakeLists.txt", "patches/**" + generators = "cmake", "cmake_find_package" + settings = "os", "compiler", "build_type", "arch" + options = { + "shared": [True, False], + "fPIC": [True, False], + "with_deprecated": [True, False], + "with_cli": [True, False], + "with_compute": [True, False], + "with_csv": [True, False], + "with_cuda": [True, False], + "with_hdfs_bridge": [True, False], + "with_dataset_modules": [True, False], + "with_filesystem_layer": [True, False], + "with_flight_rpc": [True, False], + "with_gandiva": [True, False], + "with_glog": [True, False], + "with_backtrace": [True, False], + "with_json": [True, False], + "with_openssl": [True, False], + "with_parquet": [True, False], + "with_s3": [True, False], + "with_sse42": [True, False], + "with_simd": [True, False], + "with_brotli": [True, False], + "with_bz2": [True, False], + "with_lz4": [True, False], + "with_snappy": [True, False], + "with_zlib": [True, False], + "with_zstd": [True, False], + } + default_options = { + "shared": False, + "fPIC": True, + "with_deprecated": True, + "with_cli": False, + "with_compute": False, + "with_csv": False, + "with_cuda": False, + "with_hdfs_bridge": False, + "with_dataset_modules": False, + "with_filesystem_layer": False, + "with_flight_rpc": False, + "with_gandiva": False, + "with_glog": False, + "with_backtrace": True, + "with_json": False, + "with_openssl": True, + "with_parquet": False, + "with_s3": False, + "with_sse42": True, + "with_simd": True, + "with_brotli": False, + "with_bz2": False, + "with_lz4": False, + "with_snappy": False, + "with_zlib": False, + "with_zstd": False, + } + + _cmake = None + + @property + def _source_subfolder(self): + return "source_subfolder" + + def config_options(self): + if self.settings.os == "Windows": + del self.options.fPIC + if self.settings.arch != "x86_64": + del self.options.with_sse42 + + def configure(self): + if self.options.shared: + del self.options.fPIC + if not self.settings.build_type: + self.settings.build_type = "Release" + if self.options.with_dataset_modules and not self.options.with_compute: + raise ConanInvalidConfiguration("'with_dataset_modules' options requires 'with_compute'") + + @property + def _boost_required(self): + return self.options.with_hdfs_bridge or self.options.with_gandiva + + def build_requirements(self): + if self.options.with_parquet: + raise ConanInvalidConfiguration("CCI has no thrift recipe (yet)") + self.requires("thrift/x.y.z") + + def requirements(self): + self.requires("protobuf/3.9.1.0") + self.requires("jemalloc/5.2.1") + if self._boost_required: + self.requires("boost/1.72.0") + if self.options.with_cuda: + raise ConanInvalidConfiguration("CCI has no cuda recipe (yet)") + self.requires("cuda/x.y.z") + if self.options.with_flight_rpc: + raise ConanInvalidConfiguration("CCI has no grpc recipe (yet)") + self.requires("grpc/x.y.z") + if self.options.with_backtrace or self.options.with_gandiva: + raise ConanInvalidConfiguration("CCI has no llvm recipe (yet)") + self.requires("llvm/x.y.z") + if self.options.with_gandiva: + self.requires("re2/20200301") + if self.options.with_glog: + self.requires("glog/0.4.0") + if self.options.with_json: + self.requires("rapidjson/1.1.0") + if self.options.with_openssl: + self.requires("openssl/1.1.1d") + if self.options.with_s3: + self.requires("aws-sdk-cpp/1.7.299") + if self.options.with_brotli: + self.requires("brotli/1.0.7") + if self.options.with_bz2: + self.requires("bzip2/1.0.8") + if self.options.with_lz4: + self.requires("lz4/1.9.2") + if self.options.with_snappy: + self.requires("snappy/1.1.8") + if self.options.with_zlib: + self.requires("zlib/1.2.11") + if self.options.with_zstd: + self.requires("zstd/1.4.4") + + def source(self): + tools.get(**self.conan_data["sources"][self.version]) + extracted_dir = "arrow-apache-arrow-{}".format(self.version) + os.rename(extracted_dir, self._source_subfolder) + + def _configure_cmake(self): + if self._cmake: + return self._cmake + self._cmake = CMake(self) + self._cmake.definitions["ARROW_DEPENDENCY_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_VERBOSE_THIRDPARTY_BUILD"] = True + self._cmake.definitions["ARROW_BUILD_SHARED"] = self.options.shared + self._cmake.definitions["ARROW_BUILD_STATIC"] = not self.options.shared + self._cmake.definitions["ARROW_NO_DEPRECATED_API"] = not self.options.with_deprecated + self._cmake.definitions["ARROW_HDFS"] = self.options.with_hdfs_bridge + self._cmake.definitions["ARROW_DATASET"] = self.options.with_dataset_modules + self._cmake.definitions["ARROW_FILESYSTEM"] = self.options.with_filesystem_layer + self._cmake.definitions["ARROW_FLIGHT"] = self.options.with_flight_rpc + self._cmake.definitions["ARROW_GANDIVA"] = self.options.with_gandiva + self._cmake.definitions["ARROW_COMPUTE"] = self.options.with_compute + self._cmake.definitions["ARROW_CSV"] = self.options.with_csv + self._cmake.definitions["ARROW_CUDA"] = self.options.with_cuda + self._cmake.definitions["ARROW_JSON"] = self.options.with_json + self._cmake.definitions["ARROW_PARQUET"] = self.options.with_parquet + + self._cmake.definitions["ARROW_SSE42"] = self.options.get_safe("with_sse42") + self._cmake.definitions["ARROW_USE_SIMD"] = self.options.with_simd + # self._cmake.definitions["ARROW_BOOST_VENDORED"] = False + self._cmake.definitions["BOOST_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_PROTOBUF_USE_SHARED"] = self.options["protobuf"].shared + self._cmake.definitions["ARROW_HDFS"] = self.options.with_hdfs_bridge + self._cmake.definitions["ARROW_USE_GLOG"] = self.options.with_glog + self._cmake.definitions["GLOG_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_WITH_BACKTRACE"] = self.options.with_backtrace + self._cmake.definitions["ARROW_WITH_BROTLI"] = self.options.with_brotli + self._cmake.definitions["Brotli_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_WITH_BZ2"] = self.options.with_bz2 + self._cmake.definitions["BZip2_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_WITH_LZ4"] = self.options.with_lz4 + self._cmake.definitions["Lz4_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_WITH_SNAPPY"] = self.options.with_snappy + self._cmake.definitions["Snappy_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_WITH_ZLIB"] = self.options.with_zlib + self._cmake.definitions["ZLIB_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_WITH_ZSTD"] = self.options.with_zstd + self._cmake.definitions["ZSTD_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_USE_OPENSSL"] = self.options.with_openssl + if self.options.with_openssl: + self._cmake.definitions["OPENSSL_ROOT_DIR"] = self.deps_cpp_info["openssl"].rootpath.replace("\\", "/") + if self._boost_required: + self._cmake.definitions["ARROW_BOOST_USE_SHARED"] = self.options["boost"].shared + self._cmake.definitions["ARROW_S3"] = self.options.with_s3 + self._cmake.definitions["AWSSDK_SOURCE"] = "SYSTEM" + + self._cmake.definitions["ARROW_BUILD_UTILITIES"] = self.options.with_cli + self._cmake.definitions["ARROW_BUILD_INTEGRATION"] = False + self._cmake.definitions["ARROW_INSTALL_NAME_RPATH"] = False + self._cmake.definitions["ARROW_BUILD_EXAMPLES"] = False + self._cmake.definitions["ARROW_BUILD_TESTS"] = False + self._cmake.definitions["ARROW_ENABLE_TIMING_TESTS"] = False + self._cmake.definitions["ARROW_BUILD_BENCHMARKS"] = False + + if self.settings.compiler == "Visual Studio": + self._cmake.definitions["ARROW_USE_STATIC_CRT"] = "MT" in str(self.settings.compiler.runtime) + + self._cmake.configure() + return self._cmake + + def _patch_sources(self): + for patch in self.conan_data["patches"][self.version]: + tools.patch(**patch) + + def build(self): + self._patch_sources() + cmake = self._configure_cmake() + cmake.build() + + def package(self): + self.copy("LICENSE.txt", src=self._source_subfolder, dst="licenses") + self.copy("NOTICE.txt", src=self._source_subfolder, dst="licenses") + cmake = self._configure_cmake() + cmake.install() + + tools.rmdir(os.path.join(self.package_folder, "lib", "cmake")) + tools.rmdir(os.path.join(self.package_folder, "lib", "pkgconfig")) + tools.rmdir(os.path.join(self.package_folder, "share")) + + @property + def _libs(self): + libs = [] + if self.options.with_dataset_modules: + libs.append("arrow_dataset") + libs.append("arrow") + return libs + + def package_info(self): + self.cpp_info.libs = self._libs + if not self.options.shared: + self.cpp_info.defines = ["ARROW_STATIC"] + + self.cpp_info.names["cmake_find_package"] = "Arrow" + self.cpp_info.names["cmake_find_package_multi"] = "Arrow" + + if self.options.with_cli: + binpath = os.path.join(self.package_folder, "bin") + self.output.info("Appending PATH env var: {}".format(binpath)) + self.env_info.PATH.append(binpath) diff --git a/recipes/arrow/all/patches/0001-cmake-16.0.patch b/recipes/arrow/all/patches/0001-cmake-16.0.patch new file mode 100644 index 0000000000000..dedf30fd11b89 --- /dev/null +++ b/recipes/arrow/all/patches/0001-cmake-16.0.patch @@ -0,0 +1,139 @@ +--- cpp/CMakeLists.txt ++++ cpp/CMakeLists.txt +@@ -614,7 +614,7 @@ set(ARROW_STATIC_LINK_LIBS) + set(ARROW_STATIC_INSTALL_INTERFACE_LIBS) + + if(ARROW_USE_OPENSSL) +- set(ARROW_OPENSSL_LIBS OpenSSL::Crypto OpenSSL::SSL) ++ set(ARROW_OPENSSL_LIBS OpenSSL::OpenSSL) + list(APPEND ARROW_LINK_LIBS ${ARROW_OPENSSL_LIBS}) + list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_OPENSSL_LIBS}) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_OPENSSL_LIBS}) +@@ -622,7 +622,7 @@ endif() + + if(ARROW_WITH_BROTLI) + # Order is important for static linking +- set(ARROW_BROTLI_LIBS Brotli::brotlienc Brotli::brotlidec Brotli::brotlicommon) ++ set(ARROW_BROTLI_LIBS Brotli::Brotli) + list(APPEND ARROW_LINK_LIBS ${ARROW_BROTLI_LIBS}) + list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_BROTLI_LIBS}) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_BROTLI_LIBS}) +@@ -639,8 +639,8 @@ if(ARROW_WITH_LZ4) + endif() + + if(ARROW_WITH_SNAPPY) +- list(APPEND ARROW_STATIC_LINK_LIBS Snappy::snappy) +- list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS Snappy::snappy) ++ list(APPEND ARROW_STATIC_LINK_LIBS Snappy::Snappy) ++ list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS Snappy::Snappy) + endif() + + if(ARROW_WITH_ZLIB) +@@ -661,9 +661,9 @@ if(ARROW_ORC) + endif() + + if(ARROW_USE_GLOG) +- list(APPEND ARROW_LINK_LIBS GLOG::glog) +- list(APPEND ARROW_STATIC_LINK_LIBS GLOG::glog) +- list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS GLOG::glog) ++ list(APPEND ARROW_LINK_LIBS GLOG::GLOG) ++ list(APPEND ARROW_STATIC_LINK_LIBS GLOG::GLOG) ++ list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS GLOG::GLOG) + add_definitions("-DARROW_USE_GLOG") + endif() + +--- cpp/cmake_modules/DefineOptions.cmake ++++ cpp/cmake_modules/DefineOptions.cmake +@@ -76,7 +76,7 @@ macro(define_option_string name description default) + endmacro() + + # Top level cmake dir +-if("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}") ++if(1) + #---------------------------------------------------------------------- + set_option_category("Compile and link") + +--- cpp/cmake_modules/ThirdpartyToolchain.cmake ++++ cpp/cmake_modules/ThirdpartyToolchain.cmake +@@ -725,7 +725,7 @@ if(ARROW_WITH_SNAPPY) + # location. + # https://bugzilla.redhat.com/show_bug.cgi?id=1679727 + # https://src.fedoraproject.org/rpms/snappy/pull-request/1 +- find_package(Snappy QUIET HINTS "${CMAKE_ROOT}/Modules/") ++ find_package(Snappy REQUIRED) + if(NOT Snappy_FOUND) + find_package(SnappyAlt) + endif() +@@ -737,14 +737,14 @@ if(ARROW_WITH_SNAPPY) + elseif(Snappy_SOURCE STREQUAL "SYSTEM") + # SnappyConfig.cmake is not installed on Ubuntu/Debian + # TODO: Make a bug report upstream +- find_package(Snappy HINTS "${CMAKE_ROOT}/Modules/") ++ find_package(Snappy REQUIRED) + if(NOT Snappy_FOUND) + find_package(SnappyAlt REQUIRED) + endif() + endif() + + # TODO: Don't use global includes but rather target_include_directories +- get_target_property(SNAPPY_INCLUDE_DIRS Snappy::snappy INTERFACE_INCLUDE_DIRECTORIES) ++ get_target_property(SNAPPY_INCLUDE_DIRS Snappy::Snappy INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) + endif() + +@@ -806,7 +806,7 @@ endmacro() + if(ARROW_WITH_BROTLI) + resolve_dependency(Brotli) + # TODO: Don't use global includes but rather target_include_directories +- get_target_property(BROTLI_INCLUDE_DIR Brotli::brotlicommon ++ get_target_property(BROTLI_INCLUDE_DIR Brotli::Brotli + INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${BROTLI_INCLUDE_DIR}) + endif() +@@ -918,7 +918,7 @@ endmacro() + if(ARROW_USE_GLOG) + resolve_dependency(GLOG) + # TODO: Don't use global includes but rather target_include_directories +- get_target_property(GLOG_INCLUDE_DIR GLOG::glog INTERFACE_INCLUDE_DIRECTORIES) ++ get_target_property(GLOG_INCLUDE_DIR GLOG::GLOG INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) + endif() + +@@ -1222,6 +1222,7 @@ macro(build_protobuf) + endmacro() + + if(ARROW_WITH_PROTOBUF) ++ find_package(Protobuf REQUIRED) + if(ARROW_WITH_GRPC) + # gRPC 1.21.0 or later require Protobuf 3.7.0 or later. + set(ARROW_PROTOBUF_REQUIRED_VERSION "3.7.0") +@@ -1254,9 +1255,10 @@ if(ARROW_WITH_PROTOBUF) + set(ARROW_PROTOBUF_LIBPROTOC arrow::protobuf::libprotoc) + else() + if(NOT TARGET protobuf::libprotoc) ++ set(Protobuf_PROTOC_LIBRARY protoc) + if(PROTOBUF_PROTOC_LIBRARY AND NOT Protobuf_PROTOC_LIBRARY) + # Old CMake versions have a different casing. +- set(Protobuf_PROTOC_LIBRARY ${PROTOBUF_PROTOC_LIBRARY}) ++ set(Protobuf_PROTOC_LIBRARY protoc) + endif() + if(NOT Protobuf_PROTOC_LIBRARY) + message(FATAL_ERROR "libprotoc was set to ${Protobuf_PROTOC_LIBRARY}") +@@ -1679,7 +1681,7 @@ if(ARROW_WITH_RAPIDJSON) + elseif(RapidJSON_SOURCE STREQUAL "SYSTEM") + # Fedora packages place the package information at the wrong location. + # https://bugzilla.redhat.com/show_bug.cgi?id=1680400 +- find_package(RapidJSON ${ARROW_RAPIDJSON_REQUIRED_VERSION} HINTS "${CMAKE_ROOT}") ++ find_package(RapidJSON REQUIRED) + if(NOT RapidJSON_FOUND) + # Ubuntu / Debian don't package the CMake config + find_package(RapidJSONAlt ${ARROW_RAPIDJSON_REQUIRED_VERSION} REQUIRED) +@@ -1933,7 +1935,7 @@ if(ARROW_WITH_BZ2) + PROPERTIES IMPORTED_LOCATION "${BZIP2_LIBRARIES}" + INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_INCLUDE_DIR}") + endif() +- include_directories(SYSTEM "${BZIP2_INCLUDE_DIR}") ++ include_directories(SYSTEM "${BZip2_INCLUDE_DIR}") + endif() + + macro(build_cares) diff --git a/recipes/arrow/all/patches/0002-jemalloc.patch b/recipes/arrow/all/patches/0002-jemalloc.patch new file mode 100644 index 0000000000000..7c431d86ec080 --- /dev/null +++ b/recipes/arrow/all/patches/0002-jemalloc.patch @@ -0,0 +1,41 @@ +--- cpp/cmake_modules/ThirdpartyToolchain.cmake ++++ cpp/cmake_modules/ThirdpartyToolchain.cmake +@@ -1298,6 +1298,7 @@ endif() + # jemalloc - Unix-only high-performance allocator + + if(ARROW_JEMALLOC) ++if(0) + message(STATUS "Building (vendored) jemalloc from source") + # We only use a vendored jemalloc as we want to control its version. + # Also our build of jemalloc is specially prefixed so that it will not +@@ -1354,6 +1355,8 @@ if(ARROW_JEMALLOC) + "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src") + add_dependencies(jemalloc::jemalloc jemalloc_ep) + endif() ++ find_package(jemalloc REQUIRED) ++endif() + + # ---------------------------------------------------------------------- + # mimalloc - Cross-platform high-performance allocator, from Microsoft +--- cpp/src/arrow/CMakeLists.txt ++++ cpp/src/arrow/CMakeLists.txt +@@ -217,7 +217,7 @@ set(ARROW_TESTING_SRCS + + set(_allocator_dependencies "") # Empty list + if(ARROW_JEMALLOC) +- list(APPEND _allocator_dependencies jemalloc_ep) ++ list(APPEND _allocator_dependencies jemalloc::jemalloc) + endif() + if(ARROW_MIMALLOC) + list(APPEND _allocator_dependencies mimalloc_ep) +--- cpp/src/arrow/memory_pool.cc ++++ cpp/src/arrow/memory_pool.cc +@@ -31,7 +31,7 @@ + // Needed to support jemalloc 3 and 4 + #define JEMALLOC_MANGLE + // Explicitly link to our version of jemalloc +-#include "jemalloc_ep/dist/include/jemalloc/jemalloc.h" ++#include "jemalloc/jemalloc.h" + #endif + + #ifdef ARROW_MIMALLOC diff --git a/recipes/arrow/all/test_package/CMakeLists.txt b/recipes/arrow/all/test_package/CMakeLists.txt new file mode 100644 index 0000000000000..48b17115a09d2 --- /dev/null +++ b/recipes/arrow/all/test_package/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required(VERSION 2.8.11) +project(test_package) + +include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) +conan_basic_setup() + +add_executable(${PROJECT_NAME} test_package.cpp) +target_link_libraries(${PROJECT_NAME} ${CONAN_LIBS}) diff --git a/recipes/arrow/all/test_package/conanfile.py b/recipes/arrow/all/test_package/conanfile.py new file mode 100644 index 0000000000000..bd7165a553cf4 --- /dev/null +++ b/recipes/arrow/all/test_package/conanfile.py @@ -0,0 +1,17 @@ +from conans import ConanFile, CMake, tools +import os + + +class TestPackageConan(ConanFile): + settings = "os", "compiler", "build_type", "arch" + generators = "cmake" + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + + def test(self): + if not tools.cross_building(self.settings): + bin_path = os.path.join("bin", "test_package") + self.run(bin_path, run_environment=True) diff --git a/recipes/arrow/all/test_package/test_package.cpp b/recipes/arrow/all/test_package/test_package.cpp new file mode 100644 index 0000000000000..42cab6cc76e7a --- /dev/null +++ b/recipes/arrow/all/test_package/test_package.cpp @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include +#include +#include + +#include + +using arrow::DoubleBuilder; +using arrow::Int64Builder; +using arrow::ListBuilder; + +// While we want to use columnar data structures to build efficient operations, we +// often receive data in a row-wise fashion from other systems. In the following, +// we want give a brief introduction into the classes provided by Apache Arrow by +// showing how to transform row-wise data into a columnar table. +// +// The data in this example is stored in the following struct: +struct data_row { + int64_t id; + double cost; + std::vector cost_components; +}; + +// Transforming a vector of structs into a columnar Table. +// +// The final representation should be an `arrow::Table` which in turn +// is made up of an `arrow::Schema` and a list of +// `arrow::ChunkedArray` instances. As the first step, we will iterate +// over the data and build up the arrays incrementally. For this +// task, we provide `arrow::ArrayBuilder` classes that help in the +// construction of the final `arrow::Array` instances. +// +// For each type, Arrow has a specially typed builder class. For the primitive +// values `id` and `cost` we can use the respective `arrow::Int64Builder` and +// `arrow::DoubleBuilder`. For the `cost_components` vector, we need to have two +// builders, a top-level `arrow::ListBuilder` that builds the array of offsets and +// a nested `arrow::DoubleBuilder` that constructs the underlying values array that +// is referenced by the offsets in the former array. +arrow::Status VectorToColumnarTable(const std::vector& rows, + std::shared_ptr* table) { + // The builders are more efficient using + // arrow::jemalloc::MemoryPool::default_pool() as this can increase the size of + // the underlying memory regions in-place. At the moment, arrow::jemalloc is only + // supported on Unix systems, not Windows. + arrow::MemoryPool* pool = arrow::default_memory_pool(); + + Int64Builder id_builder(pool); + DoubleBuilder cost_builder(pool); + ListBuilder components_builder(pool, std::make_shared(pool)); + // The following builder is owned by components_builder. + DoubleBuilder& cost_components_builder = + *(static_cast(components_builder.value_builder())); + + // Now we can loop over our existing data and insert it into the builders. The + // `Append` calls here may fail (e.g. we cannot allocate enough additional memory). + // Thus we need to check their return values. For more information on these values, + // check the documentation about `arrow::Status`. + for (const data_row& row : rows) { + ARROW_RETURN_NOT_OK(id_builder.Append(row.id)); + ARROW_RETURN_NOT_OK(cost_builder.Append(row.cost)); + + // Indicate the start of a new list row. This will memorise the current + // offset in the values builder. + ARROW_RETURN_NOT_OK(components_builder.Append()); + // Store the actual values. The final nullptr argument tells the underyling + // builder that all added values are valid, i.e. non-null. + ARROW_RETURN_NOT_OK(cost_components_builder.AppendValues(row.cost_components.data(), + row.cost_components.size())); + } + + // At the end, we finalise the arrays, declare the (type) schema and combine them + // into a single `arrow::Table`: + std::shared_ptr id_array; + ARROW_RETURN_NOT_OK(id_builder.Finish(&id_array)); + std::shared_ptr cost_array; + ARROW_RETURN_NOT_OK(cost_builder.Finish(&cost_array)); + // No need to invoke cost_components_builder.Finish because it is implied by + // the parent builder's Finish invocation. + std::shared_ptr cost_components_array; + ARROW_RETURN_NOT_OK(components_builder.Finish(&cost_components_array)); + + std::vector> schema_vector = { + arrow::field("id", arrow::int64()), arrow::field("cost", arrow::float64()), + arrow::field("cost_components", arrow::list(arrow::float64()))}; + + auto schema = std::make_shared(schema_vector); + + // The final `table` variable is the one we then can pass on to other functions + // that can consume Apache Arrow memory structures. This object has ownership of + // all referenced data, thus we don't have to care about undefined references once + // we leave the scope of the function building the table and its underlying arrays. + *table = arrow::Table::Make(schema, {id_array, cost_array, cost_components_array}); + + return arrow::Status::OK(); +} + +arrow::Status ColumnarTableToVector(const std::shared_ptr& table, + std::vector* rows) { + // To convert an Arrow table back into the same row-wise representation as in the + // above section, we first will check that the table conforms to our expected + // schema and then will build up the vector of rows incrementally. + // + // For the check if the table is as expected, we can utilise solely its schema. + std::vector> schema_vector = { + arrow::field("id", arrow::int64()), arrow::field("cost", arrow::float64()), + arrow::field("cost_components", arrow::list(arrow::float64()))}; + auto expected_schema = std::make_shared(schema_vector); + + if (!expected_schema->Equals(*table->schema())) { + // The table doesn't have the expected schema thus we cannot directly + // convert it to our target representation. + return arrow::Status::Invalid("Schemas are not matching!"); + } + + // As we have ensured that the table has the expected structure, we can unpack the + // underlying arrays. For the primitive columns `id` and `cost` we can use the high + // level functions to get the values whereas for the nested column + // `cost_components` we need to access the C-pointer to the data to copy its + // contents into the resulting `std::vector`. Here we need to be care to + // also add the offset to the pointer. This offset is needed to enable zero-copy + // slicing operations. While this could be adjusted automatically for double + // arrays, this cannot be done for the accompanying bitmap as often the slicing + // border would be inside a byte. + + auto ids = + std::static_pointer_cast(table->column(0)->chunk(0)); + auto costs = + std::static_pointer_cast(table->column(1)->chunk(0)); + auto cost_components = + std::static_pointer_cast(table->column(2)->chunk(0)); + auto cost_components_values = + std::static_pointer_cast(cost_components->values()); + // To enable zero-copy slices, the native values pointer might need to account + // for this slicing offset. This is not needed for the higher level functions + // like Value(…) that already account for this offset internally. + const double* ccv_ptr = cost_components_values->data()->GetValues(1); + + for (int64_t i = 0; i < table->num_rows(); i++) { + // Another simplification in this example is that we assume that there are + // no null entries, e.g. each row is fill with valid values. + int64_t id = ids->Value(i); + double cost = costs->Value(i); + const double* first = ccv_ptr + cost_components->value_offset(i); + const double* last = ccv_ptr + cost_components->value_offset(i + 1); + std::vector components_vec(first, last); + rows->push_back({id, cost, components_vec}); + } + + return arrow::Status::OK(); +} + +#define EXIT_ON_FAILURE(expr) \ + do { \ + arrow::Status status_ = (expr); \ + if (!status_.ok()) { \ + std::cerr << status_.message() << std::endl; \ + return EXIT_FAILURE; \ + } \ + } while (0); + +int main(int argc, char** argv) { + std::vector rows = { + {1, 1.0, {1.0}}, {2, 2.0, {1.0, 2.0}}, {3, 3.0, {1.0, 2.0, 3.0}}}; + + std::shared_ptr table; + EXIT_ON_FAILURE(VectorToColumnarTable(rows, &table)); + + std::vector expected_rows; + EXIT_ON_FAILURE(ColumnarTableToVector(table, &expected_rows)); + + assert(rows.size() == expected_rows.size()); + + return EXIT_SUCCESS; +} diff --git a/recipes/arrow/config.yml b/recipes/arrow/config.yml new file mode 100644 index 0000000000000..e5e40881d5003 --- /dev/null +++ b/recipes/arrow/config.yml @@ -0,0 +1,3 @@ +versions: + "0.16.0": + folder: all From f22909009e61dfd3bb0796e85e180767e1090bb9 Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Wed, 25 Mar 2020 01:01:46 +0100 Subject: [PATCH 02/16] arrow: disable with_backtrace option by default --- recipes/arrow/all/conanfile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py index eafd1108b39d0..220b1b8964fd3 100644 --- a/recipes/arrow/all/conanfile.py +++ b/recipes/arrow/all/conanfile.py @@ -55,7 +55,7 @@ class ArrowConan(ConanFile): "with_flight_rpc": False, "with_gandiva": False, "with_glog": False, - "with_backtrace": True, + "with_backtrace": False, "with_json": False, "with_openssl": True, "with_parquet": False, From e383fd4bb1c78b95130d1c69703b457f690139e6 Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Sun, 26 Apr 2020 02:47:14 +0200 Subject: [PATCH 03/16] arrow: bump to version 0.17.0 + bump openssl dependency + fix static MSVC library name --- recipes/arrow/all/conandata.yml | 12 +++---- recipes/arrow/all/conanfile.py | 9 +++-- ...ake-16.0.patch => 0001-cmake-0.17.0.patch} | 36 +++++++++---------- ...alloc.patch => 0002-jemalloc-0.17.0.patch} | 8 ++--- recipes/arrow/config.yml | 2 +- 5 files changed, 35 insertions(+), 32 deletions(-) rename recipes/arrow/all/patches/{0001-cmake-16.0.patch => 0001-cmake-0.17.0.patch} (86%) rename recipes/arrow/all/patches/{0002-jemalloc.patch => 0002-jemalloc-0.17.0.patch} (92%) diff --git a/recipes/arrow/all/conandata.yml b/recipes/arrow/all/conandata.yml index 3843f7beb5dfd..fe0140a4aa721 100644 --- a/recipes/arrow/all/conandata.yml +++ b/recipes/arrow/all/conandata.yml @@ -1,10 +1,10 @@ sources: - "0.16.0": - url: "https://github.com/apache/arrow/archive/apache-arrow-0.16.0.tar.gz" - sha256: "d7b3838758a365c8c47d55ab0df1006a70db951c6964440ba354f81f518b8d8d" + "0.17.0": + url: "https://github.com/apache/arrow/archive/apache-arrow-0.17.0.tar.gz" + sha256: "4db2233c25d1ef14f90f9de8e9d808a2d386c67e7116405ddd22d8f981fe66c1" patches: - "0.16.0": + "0.17.0": - base_path: "source_subfolder" - patch_file: "patches/0001-cmake-16.0.patch" + patch_file: "patches/0001-cmake-0.17.0.patch" - base_path: "source_subfolder" - patch_file: "patches/0002-jemalloc.patch" + patch_file: "patches/0002-jemalloc-0.17.0.patch" diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py index 220b1b8964fd3..3cb80b652ec5d 100644 --- a/recipes/arrow/all/conanfile.py +++ b/recipes/arrow/all/conanfile.py @@ -100,7 +100,7 @@ def build_requirements(self): self.requires("thrift/x.y.z") def requirements(self): - self.requires("protobuf/3.9.1.0") + self.requires("protobuf/3.9.1") self.requires("jemalloc/5.2.1") if self._boost_required: self.requires("boost/1.72.0") @@ -120,7 +120,7 @@ def requirements(self): if self.options.with_json: self.requires("rapidjson/1.1.0") if self.options.with_openssl: - self.requires("openssl/1.1.1d") + self.requires("openssl/1.1.1g") if self.options.with_s3: self.requires("aws-sdk-cpp/1.7.299") if self.options.with_brotli: @@ -228,7 +228,10 @@ def _libs(self): libs = [] if self.options.with_dataset_modules: libs.append("arrow_dataset") - libs.append("arrow") + if self.settings.compiler == "Visual Studio" and not self.options.shared: + libs.append("arrow_static") + else: + libs.append("arrow") return libs def package_info(self): diff --git a/recipes/arrow/all/patches/0001-cmake-16.0.patch b/recipes/arrow/all/patches/0001-cmake-0.17.0.patch similarity index 86% rename from recipes/arrow/all/patches/0001-cmake-16.0.patch rename to recipes/arrow/all/patches/0001-cmake-0.17.0.patch index dedf30fd11b89..cd7db374ddaa1 100644 --- a/recipes/arrow/all/patches/0001-cmake-16.0.patch +++ b/recipes/arrow/all/patches/0001-cmake-0.17.0.patch @@ -1,6 +1,6 @@ --- cpp/CMakeLists.txt +++ cpp/CMakeLists.txt -@@ -614,7 +614,7 @@ set(ARROW_STATIC_LINK_LIBS) +@@ -636,7 +636,7 @@ set(ARROW_STATIC_LINK_LIBS) set(ARROW_STATIC_INSTALL_INTERFACE_LIBS) if(ARROW_USE_OPENSSL) @@ -9,7 +9,7 @@ list(APPEND ARROW_LINK_LIBS ${ARROW_OPENSSL_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_OPENSSL_LIBS}) list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_OPENSSL_LIBS}) -@@ -622,7 +622,7 @@ endif() +@@ -644,7 +644,7 @@ endif() if(ARROW_WITH_BROTLI) # Order is important for static linking @@ -18,7 +18,7 @@ list(APPEND ARROW_LINK_LIBS ${ARROW_BROTLI_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_BROTLI_LIBS}) list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_BROTLI_LIBS}) -@@ -639,8 +639,8 @@ if(ARROW_WITH_LZ4) +@@ -661,8 +661,8 @@ if(ARROW_WITH_LZ4) endif() if(ARROW_WITH_SNAPPY) @@ -29,13 +29,13 @@ endif() if(ARROW_WITH_ZLIB) -@@ -661,9 +661,9 @@ if(ARROW_ORC) +@@ -683,9 +683,9 @@ if(ARROW_ORC) endif() if(ARROW_USE_GLOG) -- list(APPEND ARROW_LINK_LIBS GLOG::glog) -- list(APPEND ARROW_STATIC_LINK_LIBS GLOG::glog) -- list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS GLOG::glog) +- list(APPEND ARROW_LINK_LIBS glog::glog) +- list(APPEND ARROW_STATIC_LINK_LIBS glog::glog) +- list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS glog::glog) + list(APPEND ARROW_LINK_LIBS GLOG::GLOG) + list(APPEND ARROW_STATIC_LINK_LIBS GLOG::GLOG) + list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS GLOG::GLOG) @@ -55,7 +55,7 @@ --- cpp/cmake_modules/ThirdpartyToolchain.cmake +++ cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -725,7 +725,7 @@ if(ARROW_WITH_SNAPPY) +@@ -792,7 +792,7 @@ if(ARROW_WITH_SNAPPY) # location. # https://bugzilla.redhat.com/show_bug.cgi?id=1679727 # https://src.fedoraproject.org/rpms/snappy/pull-request/1 @@ -64,7 +64,7 @@ if(NOT Snappy_FOUND) find_package(SnappyAlt) endif() -@@ -737,14 +737,14 @@ if(ARROW_WITH_SNAPPY) +@@ -804,14 +804,14 @@ if(ARROW_WITH_SNAPPY) elseif(Snappy_SOURCE STREQUAL "SYSTEM") # SnappyConfig.cmake is not installed on Ubuntu/Debian # TODO: Make a bug report upstream @@ -81,7 +81,7 @@ include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) endif() -@@ -806,7 +806,7 @@ endmacro() +@@ -873,7 +873,7 @@ endmacro() if(ARROW_WITH_BROTLI) resolve_dependency(Brotli) # TODO: Don't use global includes but rather target_include_directories @@ -90,36 +90,36 @@ INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${BROTLI_INCLUDE_DIR}) endif() -@@ -918,7 +918,7 @@ endmacro() +@@ -985,7 +985,7 @@ endmacro() if(ARROW_USE_GLOG) resolve_dependency(GLOG) # TODO: Don't use global includes but rather target_include_directories -- get_target_property(GLOG_INCLUDE_DIR GLOG::glog INTERFACE_INCLUDE_DIRECTORIES) +- get_target_property(GLOG_INCLUDE_DIR glog::glog INTERFACE_INCLUDE_DIRECTORIES) + get_target_property(GLOG_INCLUDE_DIR GLOG::GLOG INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) endif() -@@ -1222,6 +1222,7 @@ macro(build_protobuf) +@@ -1222,6 +1222,6 @@ macro(build_protobuf) endmacro() - +- if(ARROW_WITH_PROTOBUF) + find_package(Protobuf REQUIRED) if(ARROW_WITH_GRPC) # gRPC 1.21.0 or later require Protobuf 3.7.0 or later. set(ARROW_PROTOBUF_REQUIRED_VERSION "3.7.0") -@@ -1254,9 +1255,10 @@ if(ARROW_WITH_PROTOBUF) +@@ -1254,9 +1254,9 @@ if(ARROW_WITH_PROTOBUF) set(ARROW_PROTOBUF_LIBPROTOC arrow::protobuf::libprotoc) else() if(NOT TARGET protobuf::libprotoc) + set(Protobuf_PROTOC_LIBRARY protoc) if(PROTOBUF_PROTOC_LIBRARY AND NOT Protobuf_PROTOC_LIBRARY) - # Old CMake versions have a different casing. +- # Old CMake versions have a different casing. - set(Protobuf_PROTOC_LIBRARY ${PROTOBUF_PROTOC_LIBRARY}) + set(Protobuf_PROTOC_LIBRARY protoc) endif() if(NOT Protobuf_PROTOC_LIBRARY) message(FATAL_ERROR "libprotoc was set to ${Protobuf_PROTOC_LIBRARY}") -@@ -1679,7 +1681,7 @@ if(ARROW_WITH_RAPIDJSON) +@@ -1682,7 +1682,7 @@ if(ARROW_WITH_RAPIDJSON) elseif(RapidJSON_SOURCE STREQUAL "SYSTEM") # Fedora packages place the package information at the wrong location. # https://bugzilla.redhat.com/show_bug.cgi?id=1680400 @@ -128,7 +128,7 @@ if(NOT RapidJSON_FOUND) # Ubuntu / Debian don't package the CMake config find_package(RapidJSONAlt ${ARROW_RAPIDJSON_REQUIRED_VERSION} REQUIRED) -@@ -1933,7 +1935,7 @@ if(ARROW_WITH_BZ2) +@@ -1938,7 +1938,7 @@ if(ARROW_WITH_BZ2) PROPERTIES IMPORTED_LOCATION "${BZIP2_LIBRARIES}" INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_INCLUDE_DIR}") endif() diff --git a/recipes/arrow/all/patches/0002-jemalloc.patch b/recipes/arrow/all/patches/0002-jemalloc-0.17.0.patch similarity index 92% rename from recipes/arrow/all/patches/0002-jemalloc.patch rename to recipes/arrow/all/patches/0002-jemalloc-0.17.0.patch index 7c431d86ec080..7b0b78a138297 100644 --- a/recipes/arrow/all/patches/0002-jemalloc.patch +++ b/recipes/arrow/all/patches/0002-jemalloc-0.17.0.patch @@ -1,14 +1,14 @@ --- cpp/cmake_modules/ThirdpartyToolchain.cmake +++ cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -1298,6 +1298,7 @@ endif() +@@ -1296,6 +1296,6 @@ endif() # jemalloc - Unix-only high-performance allocator - +- if(ARROW_JEMALLOC) +if(0) message(STATUS "Building (vendored) jemalloc from source") # We only use a vendored jemalloc as we want to control its version. # Also our build of jemalloc is specially prefixed so that it will not -@@ -1354,6 +1355,8 @@ if(ARROW_JEMALLOC) +@@ -1352,6 +1352,8 @@ if(ARROW_JEMALLOC) "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src") add_dependencies(jemalloc::jemalloc jemalloc_ep) endif() @@ -19,7 +19,7 @@ # mimalloc - Cross-platform high-performance allocator, from Microsoft --- cpp/src/arrow/CMakeLists.txt +++ cpp/src/arrow/CMakeLists.txt -@@ -217,7 +217,7 @@ set(ARROW_TESTING_SRCS +@@ -270,7 +270,7 @@ set(ARROW_TESTING_SRCS set(_allocator_dependencies "") # Empty list if(ARROW_JEMALLOC) diff --git a/recipes/arrow/config.yml b/recipes/arrow/config.yml index e5e40881d5003..60a00f1b4d579 100644 --- a/recipes/arrow/config.yml +++ b/recipes/arrow/config.yml @@ -1,3 +1,3 @@ versions: - "0.16.0": + "0.17.0": folder: all From 76a22dcac4f07dd315c3df10e37c767aa0b73b15 Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Sun, 26 Apr 2020 02:48:40 +0200 Subject: [PATCH 04/16] arrow: do not set build_type in configure() Co-Authored-By: Uilian Ries --- recipes/arrow/all/conanfile.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py index 3cb80b652ec5d..c6708bdf7bcba 100644 --- a/recipes/arrow/all/conanfile.py +++ b/recipes/arrow/all/conanfile.py @@ -85,8 +85,6 @@ def config_options(self): def configure(self): if self.options.shared: del self.options.fPIC - if not self.settings.build_type: - self.settings.build_type = "Release" if self.options.with_dataset_modules and not self.options.with_compute: raise ConanInvalidConfiguration("'with_dataset_modules' options requires 'with_compute'") From d131d499f2caaaa8ee3e94cd84cb74a832cd4f7f Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Mon, 27 Apr 2020 19:44:32 +0200 Subject: [PATCH 05/16] arrow: bump protobuf + jemalloc is optional now + check jemalloc options --- recipes/arrow/all/conanfile.py | 14 ++++++++++++-- .../arrow/all/patches/0002-jemalloc-0.17.0.patch | 14 +++++++++----- recipes/arrow/all/test_package/CMakeLists.txt | 2 ++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py index c6708bdf7bcba..e678531ae9dd5 100644 --- a/recipes/arrow/all/conanfile.py +++ b/recipes/arrow/all/conanfile.py @@ -16,6 +16,7 @@ class ArrowConan(ConanFile): options = { "shared": [True, False], "fPIC": [True, False], + "with_jemalloc": [True, False], "with_deprecated": [True, False], "with_cli": [True, False], "with_compute": [True, False], @@ -44,6 +45,7 @@ class ArrowConan(ConanFile): default_options = { "shared": False, "fPIC": True, + "with_jemalloc": True, "with_deprecated": True, "with_cli": False, "with_compute": False, @@ -85,6 +87,8 @@ def config_options(self): def configure(self): if self.options.shared: del self.options.fPIC + if self.options["jemalloc"].enable_cxx: + raise ConanInvalidConfiguration("jemmalloc.enable_cxx must be disabled") if self.options.with_dataset_modules and not self.options.with_compute: raise ConanInvalidConfiguration("'with_dataset_modules' options requires 'with_compute'") @@ -98,8 +102,9 @@ def build_requirements(self): self.requires("thrift/x.y.z") def requirements(self): - self.requires("protobuf/3.9.1") - self.requires("jemalloc/5.2.1") + self.requires("protobuf/3.11.4") + if self.options.with_jemalloc: + self.requires("jemalloc/5.2.1") if self._boost_required: self.requires("boost/1.72.0") if self.options.with_cuda: @@ -156,6 +161,7 @@ def _configure_cmake(self): self._cmake.definitions["ARROW_COMPUTE"] = self.options.with_compute self._cmake.definitions["ARROW_CSV"] = self.options.with_csv self._cmake.definitions["ARROW_CUDA"] = self.options.with_cuda + self._cmake.definitions["ARROW_JEMALLOC"] = self.options.with_jemalloc self._cmake.definitions["ARROW_JSON"] = self.options.with_json self._cmake.definitions["ARROW_PARQUET"] = self.options.with_parquet @@ -207,6 +213,10 @@ def _patch_sources(self): tools.patch(**patch) def build(self): + if self.options.shared: + if self.options["jemalloc"].enable_cxx: + raise ConanInvalidConfiguration("jemmalloc.enable_cxx must be disabled") + self._patch_sources() cmake = self._configure_cmake() cmake.build() diff --git a/recipes/arrow/all/patches/0002-jemalloc-0.17.0.patch b/recipes/arrow/all/patches/0002-jemalloc-0.17.0.patch index 7b0b78a138297..9659900878bc3 100644 --- a/recipes/arrow/all/patches/0002-jemalloc-0.17.0.patch +++ b/recipes/arrow/all/patches/0002-jemalloc-0.17.0.patch @@ -1,6 +1,8 @@ --- cpp/cmake_modules/ThirdpartyToolchain.cmake +++ cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -1296,6 +1296,6 @@ endif() +@@ -1294,8 +1294,8 @@ + + # ---------------------------------------------------------------------- # jemalloc - Unix-only high-performance allocator - if(ARROW_JEMALLOC) @@ -8,18 +10,20 @@ message(STATUS "Building (vendored) jemalloc from source") # We only use a vendored jemalloc as we want to control its version. # Also our build of jemalloc is specially prefixed so that it will not -@@ -1352,6 +1352,8 @@ if(ARROW_JEMALLOC) +@@ -1352,7 +1352,9 @@ "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src") add_dependencies(jemalloc::jemalloc jemalloc_ep) - endif() ++else(0) + find_package(jemalloc REQUIRED) +endif() - + endif() +- # ---------------------------------------------------------------------- # mimalloc - Cross-platform high-performance allocator, from Microsoft + --- cpp/src/arrow/CMakeLists.txt +++ cpp/src/arrow/CMakeLists.txt -@@ -270,7 +270,7 @@ set(ARROW_TESTING_SRCS +@@ -270,7 +270,7 @@ set(_allocator_dependencies "") # Empty list if(ARROW_JEMALLOC) diff --git a/recipes/arrow/all/test_package/CMakeLists.txt b/recipes/arrow/all/test_package/CMakeLists.txt index 48b17115a09d2..e97294f94b67d 100644 --- a/recipes/arrow/all/test_package/CMakeLists.txt +++ b/recipes/arrow/all/test_package/CMakeLists.txt @@ -6,3 +6,5 @@ conan_basic_setup() add_executable(${PROJECT_NAME} test_package.cpp) target_link_libraries(${PROJECT_NAME} ${CONAN_LIBS}) +set_property(TARGET ${PROJECT_NAME} PROPERTY CXX_STANDARD 11) +target_compile_definitions(${PROJECT_NAME} PRIVATE WITH_JEMALLOC) From 0e279be7d8035203268dc7068b3576800c28cc1b Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Mon, 27 Apr 2020 19:51:24 +0200 Subject: [PATCH 06/16] arrow: enable_cxx is only required on a static jemalloc --- recipes/arrow/all/conanfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py index e678531ae9dd5..6b9b8f0cee6d0 100644 --- a/recipes/arrow/all/conanfile.py +++ b/recipes/arrow/all/conanfile.py @@ -214,8 +214,8 @@ def _patch_sources(self): def build(self): if self.options.shared: - if self.options["jemalloc"].enable_cxx: - raise ConanInvalidConfiguration("jemmalloc.enable_cxx must be disabled") + if not self.options["jemalloc"] and self.options["jemalloc"].enable_cxx: + raise ConanInvalidConfiguration("jemmalloc.enable_cxx of a static jemalloc must be disabled") self._patch_sources() cmake = self._configure_cmake() From 6e53ea6d6c51898c364f9f973cb860a5597e86d0 Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Mon, 27 Apr 2020 20:08:24 +0200 Subject: [PATCH 07/16] jemalloc: check shared status of jemalloc dependency --- recipes/arrow/all/conanfile.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py index 6b9b8f0cee6d0..0cfc1dd022df0 100644 --- a/recipes/arrow/all/conanfile.py +++ b/recipes/arrow/all/conanfile.py @@ -87,8 +87,6 @@ def config_options(self): def configure(self): if self.options.shared: del self.options.fPIC - if self.options["jemalloc"].enable_cxx: - raise ConanInvalidConfiguration("jemmalloc.enable_cxx must be disabled") if self.options.with_dataset_modules and not self.options.with_compute: raise ConanInvalidConfiguration("'with_dataset_modules' options requires 'with_compute'") @@ -214,7 +212,7 @@ def _patch_sources(self): def build(self): if self.options.shared: - if not self.options["jemalloc"] and self.options["jemalloc"].enable_cxx: + if not self.options["jemalloc"].shared and self.options["jemalloc"].enable_cxx: raise ConanInvalidConfiguration("jemmalloc.enable_cxx of a static jemalloc must be disabled") self._patch_sources() From e45e5fbbdb9976789a242df83d8c4e540ba0683f Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Tue, 28 Apr 2020 20:25:20 +0200 Subject: [PATCH 08/16] arrow: pass cmake ARROW_USE_STATIC_CRT definition --- recipes/arrow/all/conanfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py index 0cfc1dd022df0..4a4abca3fa3ab 100644 --- a/recipes/arrow/all/conanfile.py +++ b/recipes/arrow/all/conanfile.py @@ -146,6 +146,7 @@ def _configure_cmake(self): if self._cmake: return self._cmake self._cmake = CMake(self) + self._cmake.definitions["ARROW_USE_STATIC_CRT"] = "MT" in str(self.settings.compiler.runtime) self._cmake.definitions["ARROW_DEPENDENCY_SOURCE"] = "SYSTEM" self._cmake.definitions["ARROW_VERBOSE_THIRDPARTY_BUILD"] = True self._cmake.definitions["ARROW_BUILD_SHARED"] = self.options.shared @@ -187,8 +188,7 @@ def _configure_cmake(self): self._cmake.definitions["ARROW_USE_OPENSSL"] = self.options.with_openssl if self.options.with_openssl: self._cmake.definitions["OPENSSL_ROOT_DIR"] = self.deps_cpp_info["openssl"].rootpath.replace("\\", "/") - if self._boost_required: - self._cmake.definitions["ARROW_BOOST_USE_SHARED"] = self.options["boost"].shared + self._cmake.definitions["ARROW_BOOST_USE_SHARED"] = self._boost_required and self.options["boost"].shared self._cmake.definitions["ARROW_S3"] = self.options.with_s3 self._cmake.definitions["AWSSDK_SOURCE"] = "SYSTEM" From cd7da504031ef8db47c25ee48dbfbf51d19e2fad Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Mon, 4 May 2020 17:35:50 +0200 Subject: [PATCH 09/16] arrow: compiler.runtime is only available for Visual Studio --- recipes/arrow/all/conanfile.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py index 4a4abca3fa3ab..5cf82a6b3f864 100644 --- a/recipes/arrow/all/conanfile.py +++ b/recipes/arrow/all/conanfile.py @@ -146,7 +146,8 @@ def _configure_cmake(self): if self._cmake: return self._cmake self._cmake = CMake(self) - self._cmake.definitions["ARROW_USE_STATIC_CRT"] = "MT" in str(self.settings.compiler.runtime) + if self.settings.compiler == "Visual Studio": + self._cmake.definitions["ARROW_USE_STATIC_CRT"] = "MT" in str(self.settings.compiler.runtime) self._cmake.definitions["ARROW_DEPENDENCY_SOURCE"] = "SYSTEM" self._cmake.definitions["ARROW_VERBOSE_THIRDPARTY_BUILD"] = True self._cmake.definitions["ARROW_BUILD_SHARED"] = self.options.shared From 0af78cbac308f71bbd93b78c03027cab472415a5 Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Fri, 31 Jul 2020 20:33:20 +0200 Subject: [PATCH 10/16] arrow: update to arrow/1.0.0 --- recipes/arrow/all/conandata.yml | 14 +++--- ...ke-0.17.0.patch => 1.0.0-0001-cmake.patch} | 47 +++++++------------ ...0.17.0.patch => 1.0.0-0002-jemalloc.patch} | 26 +++------- .../patches/1.0.0-0003-fix-shared-msvc.patch | 13 +++++ recipes/arrow/config.yml | 2 +- 5 files changed, 46 insertions(+), 56 deletions(-) rename recipes/arrow/all/patches/{0001-cmake-0.17.0.patch => 1.0.0-0001-cmake.patch} (76%) rename recipes/arrow/all/patches/{0002-jemalloc-0.17.0.patch => 1.0.0-0002-jemalloc.patch} (62%) create mode 100644 recipes/arrow/all/patches/1.0.0-0003-fix-shared-msvc.patch diff --git a/recipes/arrow/all/conandata.yml b/recipes/arrow/all/conandata.yml index fe0140a4aa721..49d2fe9015b84 100644 --- a/recipes/arrow/all/conandata.yml +++ b/recipes/arrow/all/conandata.yml @@ -1,10 +1,12 @@ sources: - "0.17.0": - url: "https://github.com/apache/arrow/archive/apache-arrow-0.17.0.tar.gz" - sha256: "4db2233c25d1ef14f90f9de8e9d808a2d386c67e7116405ddd22d8f981fe66c1" + "1.0.0": + url: "https://github.com/apache/arrow/archive/apache-arrow-1.0.0.tar.gz" + sha256: "08fbd4c633c08939850d619ca0224c75d7a0526467c721c0838b8aa7efccb270" patches: - "0.17.0": + "1.0.0": - base_path: "source_subfolder" - patch_file: "patches/0001-cmake-0.17.0.patch" + patch_file: "patches/1.0.0-0001-cmake.patch" - base_path: "source_subfolder" - patch_file: "patches/0002-jemalloc-0.17.0.patch" + patch_file: "patches/1.0.0-0002-jemalloc.patch" + - base_path: "source_subfolder" + patch_file: "patches/1.0.0-0003-fix-shared-msvc.patch" diff --git a/recipes/arrow/all/patches/0001-cmake-0.17.0.patch b/recipes/arrow/all/patches/1.0.0-0001-cmake.patch similarity index 76% rename from recipes/arrow/all/patches/0001-cmake-0.17.0.patch rename to recipes/arrow/all/patches/1.0.0-0001-cmake.patch index cd7db374ddaa1..573bc7f0c112f 100644 --- a/recipes/arrow/all/patches/0001-cmake-0.17.0.patch +++ b/recipes/arrow/all/patches/1.0.0-0001-cmake.patch @@ -1,15 +1,6 @@ --- cpp/CMakeLists.txt +++ cpp/CMakeLists.txt -@@ -636,7 +636,7 @@ set(ARROW_STATIC_LINK_LIBS) - set(ARROW_STATIC_INSTALL_INTERFACE_LIBS) - - if(ARROW_USE_OPENSSL) -- set(ARROW_OPENSSL_LIBS OpenSSL::Crypto OpenSSL::SSL) -+ set(ARROW_OPENSSL_LIBS OpenSSL::OpenSSL) - list(APPEND ARROW_LINK_LIBS ${ARROW_OPENSSL_LIBS}) - list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_OPENSSL_LIBS}) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_OPENSSL_LIBS}) -@@ -644,7 +644,7 @@ endif() +@@ -654,7 +654,7 @@ endif() if(ARROW_WITH_BROTLI) # Order is important for static linking @@ -17,31 +8,29 @@ + set(ARROW_BROTLI_LIBS Brotli::Brotli) list(APPEND ARROW_LINK_LIBS ${ARROW_BROTLI_LIBS}) list(APPEND ARROW_STATIC_LINK_LIBS ${ARROW_BROTLI_LIBS}) - list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${ARROW_BROTLI_LIBS}) -@@ -661,8 +661,8 @@ if(ARROW_WITH_LZ4) + endif() +@@ -668,8 +668,8 @@ if(ARROW_WITH_LZ4) endif() if(ARROW_WITH_SNAPPY) - list(APPEND ARROW_STATIC_LINK_LIBS Snappy::snappy) -- list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS Snappy::snappy) + list(APPEND ARROW_STATIC_LINK_LIBS Snappy::Snappy) -+ list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS Snappy::Snappy) endif() if(ARROW_WITH_ZLIB) -@@ -683,9 +683,9 @@ if(ARROW_ORC) + list(APPEND ARROW_STATIC_LINK_LIBS ZLIB::ZLIB) +@@ -685,9 +685,9 @@ if(ARROW_ORC) endif() if(ARROW_USE_GLOG) - list(APPEND ARROW_LINK_LIBS glog::glog) - list(APPEND ARROW_STATIC_LINK_LIBS glog::glog) -- list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS glog::glog) + list(APPEND ARROW_LINK_LIBS GLOG::GLOG) + list(APPEND ARROW_STATIC_LINK_LIBS GLOG::GLOG) -+ list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS GLOG::GLOG) add_definitions("-DARROW_USE_GLOG") endif() + if(ARROW_S3) --- cpp/cmake_modules/DefineOptions.cmake +++ cpp/cmake_modules/DefineOptions.cmake @@ -76,7 +76,7 @@ macro(define_option_string name description default) @@ -55,7 +44,7 @@ --- cpp/cmake_modules/ThirdpartyToolchain.cmake +++ cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -792,7 +792,7 @@ if(ARROW_WITH_SNAPPY) +@@ -854,7 +854,7 @@ if(ARROW_WITH_SNAPPY) # location. # https://bugzilla.redhat.com/show_bug.cgi?id=1679727 # https://src.fedoraproject.org/rpms/snappy/pull-request/1 @@ -64,7 +53,7 @@ if(NOT Snappy_FOUND) find_package(SnappyAlt) endif() -@@ -804,14 +804,14 @@ if(ARROW_WITH_SNAPPY) +@@ -866,14 +866,14 @@ if(ARROW_WITH_SNAPPY) elseif(Snappy_SOURCE STREQUAL "SYSTEM") # SnappyConfig.cmake is not installed on Ubuntu/Debian # TODO: Make a bug report upstream @@ -81,7 +70,7 @@ include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) endif() -@@ -873,7 +873,7 @@ endmacro() +@@ -938,7 +938,7 @@ endmacro() if(ARROW_WITH_BROTLI) resolve_dependency(Brotli) # TODO: Don't use global includes but rather target_include_directories @@ -90,7 +79,7 @@ INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${BROTLI_INCLUDE_DIR}) endif() -@@ -985,7 +985,7 @@ endmacro() +@@ -1057,7 +1057,7 @@ endmacro() if(ARROW_USE_GLOG) resolve_dependency(GLOG) # TODO: Don't use global includes but rather target_include_directories @@ -99,7 +88,7 @@ include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) endif() -@@ -1222,6 +1222,6 @@ macro(build_protobuf) +@@ -1329,6 +1329,6 @@ macro(build_protobuf) endmacro() - if(ARROW_WITH_PROTOBUF) @@ -107,7 +96,7 @@ if(ARROW_WITH_GRPC) # gRPC 1.21.0 or later require Protobuf 3.7.0 or later. set(ARROW_PROTOBUF_REQUIRED_VERSION "3.7.0") -@@ -1254,9 +1254,9 @@ if(ARROW_WITH_PROTOBUF) +@@ -1365,9 +1365,9 @@ if(ARROW_WITH_PROTOBUF) set(ARROW_PROTOBUF_LIBPROTOC arrow::protobuf::libprotoc) else() if(NOT TARGET protobuf::libprotoc) @@ -119,16 +108,16 @@ endif() if(NOT Protobuf_PROTOC_LIBRARY) message(FATAL_ERROR "libprotoc was set to ${Protobuf_PROTOC_LIBRARY}") -@@ -1682,7 +1682,7 @@ if(ARROW_WITH_RAPIDJSON) +@@ -1802,7 +1802,7 @@ if(ARROW_WITH_RAPIDJSON) elseif(RapidJSON_SOURCE STREQUAL "SYSTEM") # Fedora packages place the package information at the wrong location. # https://bugzilla.redhat.com/show_bug.cgi?id=1680400 - find_package(RapidJSON ${ARROW_RAPIDJSON_REQUIRED_VERSION} HINTS "${CMAKE_ROOT}") + find_package(RapidJSON REQUIRED) - if(NOT RapidJSON_FOUND) - # Ubuntu / Debian don't package the CMake config - find_package(RapidJSONAlt ${ARROW_RAPIDJSON_REQUIRED_VERSION} REQUIRED) -@@ -1938,7 +1938,7 @@ if(ARROW_WITH_BZ2) + if(RapidJSON_FOUND) + set(RAPIDJSON_INCLUDE_DIR ${RAPIDJSON_INCLUDE_DIRS}) + else() +@@ -2088,7 +2088,7 @@ if(ARROW_WITH_BZ2) PROPERTIES IMPORTED_LOCATION "${BZIP2_LIBRARIES}" INTERFACE_INCLUDE_DIRECTORIES "${BZIP2_INCLUDE_DIR}") endif() @@ -136,4 +125,4 @@ + include_directories(SYSTEM "${BZip2_INCLUDE_DIR}") endif() - macro(build_cares) + macro(build_utf8proc) diff --git a/recipes/arrow/all/patches/0002-jemalloc-0.17.0.patch b/recipes/arrow/all/patches/1.0.0-0002-jemalloc.patch similarity index 62% rename from recipes/arrow/all/patches/0002-jemalloc-0.17.0.patch rename to recipes/arrow/all/patches/1.0.0-0002-jemalloc.patch index 9659900878bc3..484f267f7b63d 100644 --- a/recipes/arrow/all/patches/0002-jemalloc-0.17.0.patch +++ b/recipes/arrow/all/patches/1.0.0-0002-jemalloc.patch @@ -1,8 +1,6 @@ --- cpp/cmake_modules/ThirdpartyToolchain.cmake +++ cpp/cmake_modules/ThirdpartyToolchain.cmake -@@ -1294,8 +1294,8 @@ - - # ---------------------------------------------------------------------- +@@ -1407,6 +1407,6 @@ endif() # jemalloc - Unix-only high-performance allocator - if(ARROW_JEMALLOC) @@ -10,28 +8,16 @@ message(STATUS "Building (vendored) jemalloc from source") # We only use a vendored jemalloc as we want to control its version. # Also our build of jemalloc is specially prefixed so that it will not -@@ -1352,7 +1352,9 @@ - "${CMAKE_CURRENT_BINARY_DIR}/jemalloc_ep-prefix/src") +@@ -1465,6 +1465,8 @@ if(ARROW_JEMALLOC) add_dependencies(jemalloc::jemalloc jemalloc_ep) -+else(0) + + list(APPEND ARROW_BUNDLED_STATIC_LIBS jemalloc::jemalloc) + endif() + find_package(jemalloc REQUIRED) +endif() - endif() -- + # ---------------------------------------------------------------------- # mimalloc - Cross-platform high-performance allocator, from Microsoft - ---- cpp/src/arrow/CMakeLists.txt -+++ cpp/src/arrow/CMakeLists.txt -@@ -270,7 +270,7 @@ - - set(_allocator_dependencies "") # Empty list - if(ARROW_JEMALLOC) -- list(APPEND _allocator_dependencies jemalloc_ep) -+ list(APPEND _allocator_dependencies jemalloc::jemalloc) - endif() - if(ARROW_MIMALLOC) - list(APPEND _allocator_dependencies mimalloc_ep) --- cpp/src/arrow/memory_pool.cc +++ cpp/src/arrow/memory_pool.cc @@ -31,7 +31,7 @@ diff --git a/recipes/arrow/all/patches/1.0.0-0003-fix-shared-msvc.patch b/recipes/arrow/all/patches/1.0.0-0003-fix-shared-msvc.patch new file mode 100644 index 0000000000000..3c7e86d5ff279 --- /dev/null +++ b/recipes/arrow/all/patches/1.0.0-0003-fix-shared-msvc.patch @@ -0,0 +1,13 @@ +--- cpp/src/arrow/CMakeLists.txt ++++ cpp/src/arrow/CMakeLists.txt +@@ -490,6 +490,10 @@ + target_compile_definitions(arrow_static PUBLIC ARROW_STATIC) + endif() + ++if(ARROW_BUILD_SHARED AND WIN32) ++target_compile_definitions(arrow_shared PRIVATE ARROW_EXPORTING) ++endif() ++ + if(ARROW_WITH_BACKTRACE) + find_package(Backtrace) + diff --git a/recipes/arrow/config.yml b/recipes/arrow/config.yml index 60a00f1b4d579..40341aa3db6cd 100644 --- a/recipes/arrow/config.yml +++ b/recipes/arrow/config.yml @@ -1,3 +1,3 @@ versions: - "0.17.0": + "1.0.0": folder: all From 993773af5915d0ebf8661f104bd8f0a0447e6559 Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Fri, 31 Jul 2020 21:29:03 +0200 Subject: [PATCH 11/16] arrow: fix jemalloc use --- recipes/arrow/all/conandata.yml | 2 ++ recipes/arrow/all/conanfile.py | 29 ++++++++++--------- .../all/patches/1.0.0-0002-jemalloc.patch | 16 ++++++++-- .../1.0.0-0004-mallctl-takes-size_t.patch | 11 +++++++ 4 files changed, 42 insertions(+), 16 deletions(-) create mode 100644 recipes/arrow/all/patches/1.0.0-0004-mallctl-takes-size_t.patch diff --git a/recipes/arrow/all/conandata.yml b/recipes/arrow/all/conandata.yml index 49d2fe9015b84..c6932a261c5fb 100644 --- a/recipes/arrow/all/conandata.yml +++ b/recipes/arrow/all/conandata.yml @@ -10,3 +10,5 @@ patches: patch_file: "patches/1.0.0-0002-jemalloc.patch" - base_path: "source_subfolder" patch_file: "patches/1.0.0-0003-fix-shared-msvc.patch" + - base_path: "source_subfolder" + patch_file: "patches/1.0.0-0004-mallctl-takes-size_t.patch" diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py index 5cf82a6b3f864..6a534db078df1 100644 --- a/recipes/arrow/all/conanfile.py +++ b/recipes/arrow/all/conanfile.py @@ -16,7 +16,7 @@ class ArrowConan(ConanFile): options = { "shared": [True, False], "fPIC": [True, False], - "with_jemalloc": [True, False], + "with_jemalloc": ["auto", True, False], "with_deprecated": [True, False], "with_cli": [True, False], "with_compute": [True, False], @@ -33,8 +33,6 @@ class ArrowConan(ConanFile): "with_openssl": [True, False], "with_parquet": [True, False], "with_s3": [True, False], - "with_sse42": [True, False], - "with_simd": [True, False], "with_brotli": [True, False], "with_bz2": [True, False], "with_lz4": [True, False], @@ -45,7 +43,7 @@ class ArrowConan(ConanFile): default_options = { "shared": False, "fPIC": True, - "with_jemalloc": True, + "with_jemalloc": "auto", "with_deprecated": True, "with_cli": False, "with_compute": False, @@ -62,8 +60,6 @@ class ArrowConan(ConanFile): "with_openssl": True, "with_parquet": False, "with_s3": False, - "with_sse42": True, - "with_simd": True, "with_brotli": False, "with_bz2": False, "with_lz4": False, @@ -78,11 +74,15 @@ class ArrowConan(ConanFile): def _source_subfolder(self): return "source_subfolder" + @property + def _with_jemalloc(self): + if self.options.with_jemalloc != "auto": + return self.options.with_jemalloc + return "BSD" in str(self.settings.os) + def config_options(self): if self.settings.os == "Windows": del self.options.fPIC - if self.settings.arch != "x86_64": - del self.options.with_sse42 def configure(self): if self.options.shared: @@ -101,7 +101,7 @@ def build_requirements(self): def requirements(self): self.requires("protobuf/3.11.4") - if self.options.with_jemalloc: + if self._with_jemalloc: self.requires("jemalloc/5.2.1") if self._boost_required: self.requires("boost/1.72.0") @@ -161,12 +161,10 @@ def _configure_cmake(self): self._cmake.definitions["ARROW_COMPUTE"] = self.options.with_compute self._cmake.definitions["ARROW_CSV"] = self.options.with_csv self._cmake.definitions["ARROW_CUDA"] = self.options.with_cuda - self._cmake.definitions["ARROW_JEMALLOC"] = self.options.with_jemalloc + self._cmake.definitions["ARROW_JEMALLOC"] = self._with_jemalloc self._cmake.definitions["ARROW_JSON"] = self.options.with_json self._cmake.definitions["ARROW_PARQUET"] = self.options.with_parquet - self._cmake.definitions["ARROW_SSE42"] = self.options.get_safe("with_sse42") - self._cmake.definitions["ARROW_USE_SIMD"] = self.options.with_simd # self._cmake.definitions["ARROW_BOOST_VENDORED"] = False self._cmake.definitions["BOOST_SOURCE"] = "SYSTEM" self._cmake.definitions["ARROW_PROTOBUF_USE_SHARED"] = self.options["protobuf"].shared @@ -212,8 +210,8 @@ def _patch_sources(self): tools.patch(**patch) def build(self): - if self.options.shared: - if not self.options["jemalloc"].shared and self.options["jemalloc"].enable_cxx: + if self.options.shared and self._with_jemalloc: + if self.options["jemalloc"].enable_cxx: raise ConanInvalidConfiguration("jemmalloc.enable_cxx of a static jemalloc must be disabled") self._patch_sources() @@ -230,6 +228,9 @@ def package(self): tools.rmdir(os.path.join(self.package_folder, "lib", "pkgconfig")) tools.rmdir(os.path.join(self.package_folder, "share")) + def package_id(self): + self.options.with_jemalloc = self._with_jemalloc + @property def _libs(self): libs = [] diff --git a/recipes/arrow/all/patches/1.0.0-0002-jemalloc.patch b/recipes/arrow/all/patches/1.0.0-0002-jemalloc.patch index 484f267f7b63d..7e8bd1ed08039 100644 --- a/recipes/arrow/all/patches/1.0.0-0002-jemalloc.patch +++ b/recipes/arrow/all/patches/1.0.0-0002-jemalloc.patch @@ -12,12 +12,24 @@ add_dependencies(jemalloc::jemalloc jemalloc_ep) list(APPEND ARROW_BUNDLED_STATIC_LIBS jemalloc::jemalloc) - endif() ++else() + find_package(jemalloc REQUIRED) +endif() - + endif() +- # ---------------------------------------------------------------------- # mimalloc - Cross-platform high-performance allocator, from Microsoft +--- cpp/src/arrow/CMakeLists.txt ++++ cpp/src/arrow/CMakeLists.txt +@@ -292,7 +292,7 @@ + + set(_allocator_dependencies "") # Empty list + if(ARROW_JEMALLOC) +- list(APPEND _allocator_dependencies jemalloc_ep) ++ list(APPEND _allocator_dependencies jemalloc::jemalloc) + endif() + if(ARROW_MIMALLOC) + list(APPEND _allocator_dependencies mimalloc_ep) --- cpp/src/arrow/memory_pool.cc +++ cpp/src/arrow/memory_pool.cc @@ -31,7 +31,7 @@ diff --git a/recipes/arrow/all/patches/1.0.0-0004-mallctl-takes-size_t.patch b/recipes/arrow/all/patches/1.0.0-0004-mallctl-takes-size_t.patch new file mode 100644 index 0000000000000..e9ce3546d355f --- /dev/null +++ b/recipes/arrow/all/patches/1.0.0-0004-mallctl-takes-size_t.patch @@ -0,0 +1,11 @@ +--- cpp/src/arrow/memory_pool.cc ++++ cpp/src/arrow/CMakeLists.txt +@@ -427,7 +427,7 @@ + + Status jemalloc_set_decay_ms(int ms) { + #ifdef ARROW_JEMALLOC +- ssize_t decay_time_ms = static_cast(ms); ++ size_t decay_time_ms = static_cast(ms); + + int err = mallctl("arenas.dirty_decay_ms", nullptr, nullptr, &decay_time_ms, + sizeof(decay_time_ms)); From 98b36479e5155a244a42aa05d6fe8b7dc5605cae Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Thu, 13 Aug 2020 02:01:14 +0200 Subject: [PATCH 12/16] arrow: add no -Werror when building Debug --- recipes/arrow/all/patches/1.0.0-0001-cmake.patch | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/recipes/arrow/all/patches/1.0.0-0001-cmake.patch b/recipes/arrow/all/patches/1.0.0-0001-cmake.patch index 573bc7f0c112f..182fcc5fb7f88 100644 --- a/recipes/arrow/all/patches/1.0.0-0001-cmake.patch +++ b/recipes/arrow/all/patches/1.0.0-0001-cmake.patch @@ -126,3 +126,14 @@ endif() macro(build_utf8proc) +--- cpp/cmake_modules/SetupCxxFlags.cmake ++++ cpp/cmake_modules/SetupCxxFlags.cmake +@@ -188,7 +188,7 @@ + message(STATUS "Arrow build warning level: ${BUILD_WARNING_LEVEL}") + + macro(arrow_add_werror_if_debug) +- if("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG") ++ if(0) + # Treat all compiler warnings as errors + if(MSVC) + set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} /WX") From f813f1d5b29973271fb85cfc645a770a49a09bfe Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Thu, 13 Aug 2020 02:09:02 +0200 Subject: [PATCH 13/16] arrow: bump required cmake version --- recipes/arrow/all/CMakeLists.txt | 2 +- recipes/arrow/all/test_package/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/arrow/all/CMakeLists.txt b/recipes/arrow/all/CMakeLists.txt index 4d1896ed77c3c..5fce337b405db 100644 --- a/recipes/arrow/all/CMakeLists.txt +++ b/recipes/arrow/all/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8.12) +cmake_minimum_required(VERSION 3.1) project(cmake_wrapper) include(conanbuildinfo.cmake) diff --git a/recipes/arrow/all/test_package/CMakeLists.txt b/recipes/arrow/all/test_package/CMakeLists.txt index e97294f94b67d..0770e4cb69ded 100644 --- a/recipes/arrow/all/test_package/CMakeLists.txt +++ b/recipes/arrow/all/test_package/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 2.8.11) +cmake_minimum_required(VERSION 3.1) project(test_package) include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) From d1c8ca3ec3cbb4b600ae00644ad044e2aaddc6ae Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Thu, 13 Aug 2020 21:44:12 +0200 Subject: [PATCH 14/16] arrow: add component support --- recipes/arrow/all/conanfile.py | 386 ++++++++++++++---- .../arrow/all/patches/1.0.0-0001-cmake.patch | 12 + 2 files changed, 314 insertions(+), 84 deletions(-) diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py index 6a534db078df1..d2405c4d135b1 100644 --- a/recipes/arrow/all/conanfile.py +++ b/recipes/arrow/all/conanfile.py @@ -16,23 +16,34 @@ class ArrowConan(ConanFile): options = { "shared": [True, False], "fPIC": [True, False], - "with_jemalloc": ["auto", True, False], - "with_deprecated": [True, False], - "with_cli": [True, False], - "with_compute": [True, False], + "gandiva": [True, False], + "parquet": [True, False], + "plasma": [True, False], + "cli": [True, False], + "compute": ["auto", True, False], + "dataset_modules": [True, False], + "deprecated": [True, False], + "encryption": [True, False], + "filesystem_layer": [True, False], + "hdfs_bridgs": [True, False], + "with_backtrace": [True, False], + "with_boost": ["auto", True, False], "with_csv": [True, False], "with_cuda": [True, False], - "with_hdfs_bridge": [True, False], - "with_dataset_modules": [True, False], - "with_filesystem_layer": [True, False], "with_flight_rpc": [True, False], - "with_gandiva": [True, False], - "with_glog": [True, False], - "with_backtrace": [True, False], + "with_gflags": ["auto", True, False], + "with_glog": ["auto", True, False], + "with_grpc": ["auto", True, False], + "with_hiveserver2": [True, False], + "with_jemalloc": ["auto", True, False], "with_json": [True, False], - "with_openssl": [True, False], - "with_parquet": [True, False], + "with_llvm": ["auto", True, False], + "with_openssl": ["auto", True, False], + "with_orc": [True, False], + "with_protobuf": ["auto", True, False], + "with_re2": ["auto", True, False], "with_s3": [True, False], + "with_utf8proc": ["auto", True, False], "with_brotli": [True, False], "with_bz2": [True, False], "with_lz4": [True, False], @@ -43,25 +54,36 @@ class ArrowConan(ConanFile): default_options = { "shared": False, "fPIC": True, - "with_jemalloc": "auto", - "with_deprecated": True, - "with_cli": False, - "with_compute": False, + "gandiva": False, + "parquet": False, + "plasma": False, + "cli": False, + "compute": "auto", + "dataset_modules": False, + "deprecated": True, + "encryption": False, + "filesystem_layer": False, + "hdfs_bridgs": False, + "with_backtrace": False, + "with_boost": "auto", + "with_brotli": False, + "with_bz2": False, "with_csv": False, "with_cuda": False, - "with_hdfs_bridge": False, - "with_dataset_modules": False, - "with_filesystem_layer": False, "with_flight_rpc": False, - "with_gandiva": False, - "with_glog": False, - "with_backtrace": False, + "with_gflags": "auto", + "with_jemalloc": "auto", + "with_glog": "auto", + "with_grpc": "auto", + "with_hiveserver2": False, "with_json": False, - "with_openssl": True, - "with_parquet": False, + "with_llvm": "auto", + "with_openssl": "auto", + "with_orc": False, + "with_protobuf": "auto", + "with_re2": "auto", "with_s3": False, - "with_brotli": False, - "with_bz2": False, + "with_utf8proc": "auto", "with_lz4": False, "with_snappy": False, "with_zlib": False, @@ -74,12 +96,6 @@ class ArrowConan(ConanFile): def _source_subfolder(self): return "source_subfolder" - @property - def _with_jemalloc(self): - if self.options.with_jemalloc != "auto": - return self.options.with_jemalloc - return "BSD" in str(self.settings.os) - def config_options(self): if self.settings.os == "Windows": del self.options.fPIC @@ -87,40 +103,129 @@ def config_options(self): def configure(self): if self.options.shared: del self.options.fPIC - if self.options.with_dataset_modules and not self.options.with_compute: - raise ConanInvalidConfiguration("'with_dataset_modules' options requires 'with_compute'") + if self.options.compute == False and not self._compute(True): + raise ConanInvalidConfiguration("compute options is required (or choose auto)") + if self.options.with_jemalloc == False and self._with_jemalloc(True): + raise ConanInvalidConfiguration("with_jemalloc option is required (or choose auto)") + if self.options.with_re2 == False and self._with_re2(True): + raise ConanInvalidConfiguration("with_re2 option is required (or choose auto)") + if self.options.with_protobuf == False and self._with_protobuf(True): + raise ConanInvalidConfiguration("with_protobuf option is required (or choose auto)") + if self.options.with_gflags == False and self._with_gflags(True): + raise ConanInvalidConfiguration("with_gflags options is required (or choose auto)") + if self.options.with_grpc == False and self._with_grpc(True): + raise ConanInvalidConfiguration("with_grpc options is required (or choose auto)") + if self.options.with_boost == False and self._with_boost(True): + raise ConanInvalidConfiguration("with_boost options is required (or choose auto)") + if self.options.with_openssl == False and self._with_openssl(True): + raise ConanInvalidConfiguration("with_openssl options is required (or choose auto)") + if self.options.with_llvm == False and self._with_llvm(True): + raise ConanInvalidConfiguration("with_openssl options is required (or choose auto)") - @property - def _boost_required(self): - return self.options.with_hdfs_bridge or self.options.with_gandiva + def _compute(self, required=False): + if required or self.options.compute == "auto": + return bool(self.options.dataset_modules) + else: + return bool(self.options.compute) + + def _with_jemalloc(self, required=False): + if required or self.options.with_jemalloc == "auto": + return bool("BSD" in str(self.settings.os)) + else: + return bool(self.options.with_jemalloc) + + def _with_re2(self, required=False): + if required or self.options.with_re2 == "auto": + return bool(self.options.gandiva) + else: + return bool(self.options.with_re2) + + def _with_protobuf(self, required=False): + if required or self.options.with_protobuf == "auto": + return bool(self.options.gandiva or self.options.with_flight_rpc or self.options.with_orc) + else: + return bool(self.options.with_protobuf) + + def _with_gflags(self, required=False): + if required or self.options.with_gflags == "auto": + return bool(self.options.plasma or self._with_glog() or self._with_grpc()) + else: + return bool(self.options.with_gflags) + + def _with_glog(self, required=False): + if required or self.options.with_glog == "auto": + return False + else: + return bool(self.options.with_glog) + + def _with_grpc(self, required=False): + if required or self.options.with_grpc == "auto": + return bool(self.options.with_flight_rpc) + else: + return bool(self.options.with_grpc) + + def _with_boost(self, required=False): + if required or self.options.with_boost == "auto": + if self.options.gandiva: + return True + if self.options.parquet and self.settings.compiler == "gcc" and self.settings.compiler.version < tools.Version("4.9"): + return True + return False + else: + return bool(self.options.with_boost) + + def _with_thrift(self, required=False): + # No self.options.with_thift exists + return bool(self.options.with_hiveserver2 or self.options.parquet) + + def _with_utf8proc(self, required=False): + if required or self.options.with_utf8proc == "auto": + return False + else: + return bool(self.options.with_utf8proc) + + def _with_llvm(self, required=False): + if required or self.options.with_llvm == "auto": + return bool(self.options.gandiva) + else: + return bool(self.options.with_openssl) + + def _with_openssl(self, required=False): + if required or self.options.with_openssl == "auto": + return bool(self.options.encryption or self.options.with_flight_rpc or self.options.with_s3) + else: + return bool(self.options.with_openssl) def build_requirements(self): - if self.options.with_parquet: + if self._with_grpc(): + raise ConanInvalidConfiguration("CCI has no grpc recipe (yet)") + if self._with_thrift(): raise ConanInvalidConfiguration("CCI has no thrift recipe (yet)") - self.requires("thrift/x.y.z") def requirements(self): - self.requires("protobuf/3.11.4") - if self._with_jemalloc: + if self.options.with_backtrace: + raise ConanInvalidConfiguration("CCI has no backtrace recipe (yet)") + if self._with_protobuf(): + self.requires("protobuf/3.11.4") + if self._with_jemalloc(): self.requires("jemalloc/5.2.1") - if self._boost_required: + if self._with_boost(): self.requires("boost/1.72.0") if self.options.with_cuda: raise ConanInvalidConfiguration("CCI has no cuda recipe (yet)") - self.requires("cuda/x.y.z") if self.options.with_flight_rpc: - raise ConanInvalidConfiguration("CCI has no grpc recipe (yet)") - self.requires("grpc/x.y.z") - if self.options.with_backtrace or self.options.with_gandiva: - raise ConanInvalidConfiguration("CCI has no llvm recipe (yet)") - self.requires("llvm/x.y.z") - if self.options.with_gandiva: - self.requires("re2/20200301") - if self.options.with_glog: + raise ConanInvalidConfiguration("CCI has no flight_rpc recipe (yet)") + if self._with_gflags(): + self.requires("gflags/2.2.2") + if self._with_glog(): self.requires("glog/0.4.0") + if self.options.with_hiveserver2: + raise ConanInvalidConfiguration("CCI has no hiveserver2 recipe (yet)") if self.options.with_json: self.requires("rapidjson/1.1.0") - if self.options.with_openssl: + if self._with_llvm(): + raise ConanInvalidConfiguration("CCI has no llvm recipe (yet)") + if self._with_openssl(): self.requires("openssl/1.1.1g") if self.options.with_s3: self.requires("aws-sdk-cpp/1.7.299") @@ -128,6 +233,8 @@ def requirements(self): self.requires("brotli/1.0.7") if self.options.with_bz2: self.requires("bzip2/1.0.8") + if self.options.with_orc: + raise ConanInvalidConfiguration("CCI has no orc recipe (yet)") if self.options.with_lz4: self.requires("lz4/1.9.2") if self.options.with_snappy: @@ -136,6 +243,8 @@ def requirements(self): self.requires("zlib/1.2.11") if self.options.with_zstd: self.requires("zstd/1.4.4") + if self._with_re2(): + self.requires("re2/20200301") def source(self): tools.get(**self.conan_data["sources"][self.version]) @@ -149,31 +258,39 @@ def _configure_cmake(self): if self.settings.compiler == "Visual Studio": self._cmake.definitions["ARROW_USE_STATIC_CRT"] = "MT" in str(self.settings.compiler.runtime) self._cmake.definitions["ARROW_DEPENDENCY_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_GANDIVA"] = self.options.gandiva + self._cmake.definitions["ARROW_PARQUET"] = self.options.parquet + self._cmake.definitions["ARROW_PLASMA"] = self.options.plasma + self._cmake.definitions["ARROW_DATASET"] = self.options.dataset_modules + self._cmake.definitions["ARROW_FILESYSTEM"] = self.options.filesystem_layer + self._cmake.definitions["PARQUET_REQUIRE_ENCRYPTION"] = self.options.encryption + self._cmake.definitions["ARROW_HDFS"] = self.options.hdfs_bridgs self._cmake.definitions["ARROW_VERBOSE_THIRDPARTY_BUILD"] = True self._cmake.definitions["ARROW_BUILD_SHARED"] = self.options.shared self._cmake.definitions["ARROW_BUILD_STATIC"] = not self.options.shared - self._cmake.definitions["ARROW_NO_DEPRECATED_API"] = not self.options.with_deprecated - self._cmake.definitions["ARROW_HDFS"] = self.options.with_hdfs_bridge - self._cmake.definitions["ARROW_DATASET"] = self.options.with_dataset_modules - self._cmake.definitions["ARROW_FILESYSTEM"] = self.options.with_filesystem_layer + self._cmake.definitions["ARROW_NO_DEPRECATED_API"] = not self.options.deprecated self._cmake.definitions["ARROW_FLIGHT"] = self.options.with_flight_rpc - self._cmake.definitions["ARROW_GANDIVA"] = self.options.with_gandiva - self._cmake.definitions["ARROW_COMPUTE"] = self.options.with_compute + self._cmake.definitions["ARROW_HIVESERVER2"] = self.options.with_hiveserver2 + self._cmake.definitions["ARROW_COMPUTE"] = self._compute() self._cmake.definitions["ARROW_CSV"] = self.options.with_csv self._cmake.definitions["ARROW_CUDA"] = self.options.with_cuda - self._cmake.definitions["ARROW_JEMALLOC"] = self._with_jemalloc + self._cmake.definitions["ARROW_JEMALLOC"] = self._with_jemalloc() self._cmake.definitions["ARROW_JSON"] = self.options.with_json - self._cmake.definitions["ARROW_PARQUET"] = self.options.with_parquet # self._cmake.definitions["ARROW_BOOST_VENDORED"] = False self._cmake.definitions["BOOST_SOURCE"] = "SYSTEM" - self._cmake.definitions["ARROW_PROTOBUF_USE_SHARED"] = self.options["protobuf"].shared - self._cmake.definitions["ARROW_HDFS"] = self.options.with_hdfs_bridge - self._cmake.definitions["ARROW_USE_GLOG"] = self.options.with_glog + self._cmake.definitions["Protobuf_SOURCE"] = "SYSTEM" + self._cmake.definitions["gRPC_SOURCE"] = "SYSTEM" + if self._with_protobuf(): + self._cmake.definitions["ARROW_PROTOBUF_USE_SHARED"] = self.options["protobuf"].shared + self._cmake.definitions["ARROW_HDFS"] = self.options.hdfs_bridgs + self._cmake.definitions["ARROW_USE_GLOG"] = self._with_glog() self._cmake.definitions["GLOG_SOURCE"] = "SYSTEM" self._cmake.definitions["ARROW_WITH_BACKTRACE"] = self.options.with_backtrace self._cmake.definitions["ARROW_WITH_BROTLI"] = self.options.with_brotli self._cmake.definitions["Brotli_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_BROTLI_USE_SHARED"] = "brotli" in self.options and not self.options["brotli"].shared + self._cmake.definitions["gflags_SOURCE"] = "SYSTEM" self._cmake.definitions["ARROW_WITH_BZ2"] = self.options.with_bz2 self._cmake.definitions["BZip2_SOURCE"] = "SYSTEM" self._cmake.definitions["ARROW_WITH_LZ4"] = self.options.with_lz4 @@ -181,17 +298,23 @@ def _configure_cmake(self): self._cmake.definitions["ARROW_WITH_SNAPPY"] = self.options.with_snappy self._cmake.definitions["Snappy_SOURCE"] = "SYSTEM" self._cmake.definitions["ARROW_WITH_ZLIB"] = self.options.with_zlib + self._cmake.definitions["RE2_SOURCE"] = "SYSTEM" self._cmake.definitions["ZLIB_SOURCE"] = "SYSTEM" self._cmake.definitions["ARROW_WITH_ZSTD"] = self.options.with_zstd self._cmake.definitions["ZSTD_SOURCE"] = "SYSTEM" - self._cmake.definitions["ARROW_USE_OPENSSL"] = self.options.with_openssl - if self.options.with_openssl: + self._cmake.definitions["ORC_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_WITH_THRIFT"] = self._with_thrift() + self._cmake.definitions["Thrift_SOURCE"] = "SYSTEM" + self._cmake.definitions["THRIFT_VERSION"] = "1.0" # a recent thrift does not require boost + self._cmake.definitions["ARROW_USE_OPENSSL"] = self._with_openssl() + if self._with_openssl(): self._cmake.definitions["OPENSSL_ROOT_DIR"] = self.deps_cpp_info["openssl"].rootpath.replace("\\", "/") - self._cmake.definitions["ARROW_BOOST_USE_SHARED"] = self._boost_required and self.options["boost"].shared + if self._with_boost(): + self._cmake.definitions["ARROW_BOOST_USE_SHARED"] = self.options["boost"].shared self._cmake.definitions["ARROW_S3"] = self.options.with_s3 self._cmake.definitions["AWSSDK_SOURCE"] = "SYSTEM" - self._cmake.definitions["ARROW_BUILD_UTILITIES"] = self.options.with_cli + self._cmake.definitions["ARROW_BUILD_UTILITIES"] = self.options.cli self._cmake.definitions["ARROW_BUILD_INTEGRATION"] = False self._cmake.definitions["ARROW_INSTALL_NAME_RPATH"] = False self._cmake.definitions["ARROW_BUILD_EXAMPLES"] = False @@ -199,9 +322,16 @@ def _configure_cmake(self): self._cmake.definitions["ARROW_ENABLE_TIMING_TESTS"] = False self._cmake.definitions["ARROW_BUILD_BENCHMARKS"] = False + self._cmake.definitions["LLVM_SOURCE"] = "SYSTEM" + self._cmake.definitions["ARROW_WITH_UTF8PROC"] = self._with_utf8proc() + self._cmake.definitions["utf8proc_SOURCE"] = "SYSTEM" + if self.settings.compiler == "Visual Studio": self._cmake.definitions["ARROW_USE_STATIC_CRT"] = "MT" in str(self.settings.compiler.runtime) + if self._with_llvm(): + self._cmake.definitions["LLVM_DIR"] = self.deps_cpp_info["llvm"].rootpath.replace("\\", "/") + self._cmake.configure() return self._cmake @@ -210,7 +340,7 @@ def _patch_sources(self): tools.patch(**patch) def build(self): - if self.options.shared and self._with_jemalloc: + if self.options.shared and self._with_jemalloc(): if self.options["jemalloc"].enable_cxx: raise ConanInvalidConfiguration("jemmalloc.enable_cxx of a static jemalloc must be disabled") @@ -228,29 +358,117 @@ def package(self): tools.rmdir(os.path.join(self.package_folder, "lib", "pkgconfig")) tools.rmdir(os.path.join(self.package_folder, "share")) - def package_id(self): - self.options.with_jemalloc = self._with_jemalloc - - @property - def _libs(self): - libs = [] - if self.options.with_dataset_modules: - libs.append("arrow_dataset") + def _lib_name(self, name): if self.settings.compiler == "Visual Studio" and not self.options.shared: - libs.append("arrow_static") + return "{}_static".format(name) else: - libs.append("arrow") - return libs + return "{}".format(name) + + def package_id(self): + self.options.with_jemalloc = self._with_jemalloc() + self.info.options.with_gflags = self._with_gflags() + self.info.options.with_protobuf = self._with_protobuf() + self.info.options.with_re2 = self._with_re2() + self.info.options.with_jemalloc = self._with_jemalloc() + self.info.options.with_openssl = self._with_openssl() + self.info.options.with_boost = self._with_boost() + self.info.options.with_glog = self._with_glog() + self.info.options.with_grpc = self._with_grpc() def package_info(self): - self.cpp_info.libs = self._libs + self.cpp_info.components["libarrow"].libs = [self._lib_name("arrow")] + self.cpp_info.components["libarrow"].filenames["cmake_find_package"] = "Arrow" + self.cpp_info.components["libarrow"].filenames["cmake_find_package_multi"] = "Arrow" + self.cpp_info.components["libarrow"].names["cmake_find_package"] = "arrow" + self.cpp_info.components["libarrow"].names["cmake_find_package_multi"] = "arrow" + self.cpp_info.components["libarrow"].names["pkg_config"] = "arrow" if not self.options.shared: - self.cpp_info.defines = ["ARROW_STATIC"] + self.cpp_info.components["libarrow"].defines = ["ARROW_STATIC"] + if self.settings.os == "Linux": + self.cpp_info.components["libarrow"].system_libs = ["pthread"] - self.cpp_info.names["cmake_find_package"] = "Arrow" - self.cpp_info.names["cmake_find_package_multi"] = "Arrow" - if self.options.with_cli: + if self.options.parquet: + self.cpp_info.components["libparquet"].libs = [self._lib_name("parquet")] + self.cpp_info.components["libparquet"].filenames["cmake_find_package"] = "Parquet" + self.cpp_info.components["libparquet"].filenames["cmake_find_package_multi"] = "Parquet" + self.cpp_info.components["libparquet"].names["cmake_find_package"] = "parquet" + self.cpp_info.components["libparquet"].names["cmake_find_package_multi"] = "parquet" + self.cpp_info.components["libparquet"].names["pkg_config"] = "parquet" + self.cpp_info.components["libparquet"].requires = ["libarrow"] + + if self.options.plasma: + self.cpp_info.components["libplasma"].libs = [self._lib_name("plasma")] + self.cpp_info.components["libplasma"].filenames["cmake_find_package"] = "Plasma" + self.cpp_info.components["libplasma"].filenames["cmake_find_package_multi"] = "Arrow" + self.cpp_info.components["libplasma"].names["cmake_find_package"] = "plasma" + self.cpp_info.components["libplasma"].names["cmake_find_package_multi"] = "plasma" + self.cpp_info.components["libplasma"].names["pkg_config"] = "plasma" + self.cpp_info.components["libplasma"].requires = ["libarrow"] + + if self.options.gandiva: + self.cpp_info.components["libgandiva"].libs = [self._lib_name("gandiva")] + self.cpp_info.components["libgandiva"].filenames["cmake_find_package"] = "Gandiva" + self.cpp_info.components["libgandiva"].filenames["cmake_find_package_multi"] = "Gandiva" + self.cpp_info.components["libgandiva"].names["cmake_find_package"] = "gandiva" + self.cpp_info.components["libgandiva"].names["cmake_find_package_multi"] = "plasma" + self.cpp_info.components["libgandiva"].names["pkg_config"] = "plasma" + self.cpp_info.components["libgandiva"].requires = ["libarrow"] + + if self.options.dataset_modules: + self.cpp_info.components["dataset"].libs = ["arrow_dataset"] + + if self.options.cli: binpath = os.path.join(self.package_folder, "bin") self.output.info("Appending PATH env var: {}".format(binpath)) self.env_info.PATH.append(binpath) + + if self._with_boost(): + if self.options.gandiva: + # FIXME: only filesystem component is used + self.cpp_info.components["libgandiva"].requires.append("boost::boost") + if self.options.parquet and self.settings.compiler == "gcc" and self.settings.compiler.version < tools.Version("4.9"): + self.cpp_info.components["libparquet"].requires.append("boost::boost") + if self._with_openssl(): + self.cpp_info.components["libarrow"].requires.append("openssl::openssl") + if self._with_gflags(): + self.cpp_info.components["libarrow"].requires.append("gflags::gflags") + if self._with_glog(): + self.cpp_info.components["libarrow"].requires.append("glog::glog") + if self._with_jemalloc(): + self.cpp_info.components["libarrow"].requires.append("jemalloc::jemalloc") + if self._with_re2(): + self.cpp_info.components["libgandiva"].requires.append("re2::re2") + if self._with_protobuf(): + self.cpp_info.components["libarrow"].requires.append("protobuf::protobuf") + if self._with_utf8proc(): + self.cpp_info.components["libarrow"].requires.append("uff8proc::uff8proc") + if self._with_llvm(): + self.cpp_info.components["libarrow"].requires.append("llvm::llvm") + + if self.options.with_backtrace: + self.cpp_info.components["libarrow"].requires.append("backtrace::backtrace") + if self.options.with_cuda: + self.cpp_info.components["libarrow"].requires.append("cuda::cuda") + if self.options.with_flight_rpc: + self.cpp_info.components["libarrow"].requires.append("flight::flight") + if self.options.with_hiveserver2: + self.cpp_info.components["libarrow"].requires.append("hiveserver2::hiveserver2") + if self.options.with_json: + self.cpp_info.components["libarrow"].requires.append("rapidjson::rapidjson") + if self.options.with_s3: + self.cpp_info.components["libarrow"].requires.append("aws-sdk-cpp::filesystem") + if self.options.with_orc: + self.cpp_info.components["libarrow"].requires.append("orc::orc") + if self.options.with_brotli: + self.cpp_info.components["libarrow"].requires.append("brotli::brotli") + if self.options.with_bz2: + self.cpp_info.components["libarrow"].requires.append("bz2::bz2") + if self.options.with_lz4: + self.cpp_info.components["libarrow"].requires.append("lz4::lz4") + if self.options.with_snappy: + self.cpp_info.components["libarrow"].requires.append("snappy::snappy") + if self.options.with_zlib: + self.cpp_info.components["libarrow"].requires.append("zlib::zlib") + if self.options.with_zstd: + self.cpp_info.components["libarrow"].requires.append("zstd::zstd") diff --git a/recipes/arrow/all/patches/1.0.0-0001-cmake.patch b/recipes/arrow/all/patches/1.0.0-0001-cmake.patch index 182fcc5fb7f88..30255e99a5007 100644 --- a/recipes/arrow/all/patches/1.0.0-0001-cmake.patch +++ b/recipes/arrow/all/patches/1.0.0-0001-cmake.patch @@ -88,6 +88,18 @@ include_directories(SYSTEM ${GLOG_INCLUDE_DIR}) endif() +@@ -1139,8 +1139,8 @@ + build_gflags() + elseif(gflags_SOURCE STREQUAL "SYSTEM") +- # gflagsConfig.cmake is not installed on Ubuntu/Debian +- # TODO: Make a bug report upstream +- find_package(gflags ${ARROW_GFLAGS_REQUIRED_VERSION}) ++ find_package(gflags REQUIRED) ++ add_library(gflags-shared INTERFACE) ++ target_link_libraries(gflags-shared INTERFACE gflags::gflags) + if(NOT gflags_FOUND) + find_package(gflagsAlt ${ARROW_GFLAGS_REQUIRED_VERSION} REQUIRED) + endif() @@ -1329,6 +1329,6 @@ macro(build_protobuf) endmacro() - From b8d9a12f792c3c4875c949f0ea86da381f9b684e Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Thu, 13 Aug 2020 22:16:06 +0200 Subject: [PATCH 15/16] arrow: thrift is available on cci --- recipes/arrow/all/conanfile.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py index d2405c4d135b1..5e7dc6b2b0327 100644 --- a/recipes/arrow/all/conanfile.py +++ b/recipes/arrow/all/conanfile.py @@ -199,10 +199,10 @@ def _with_openssl(self, required=False): def build_requirements(self): if self._with_grpc(): raise ConanInvalidConfiguration("CCI has no grpc recipe (yet)") - if self._with_thrift(): - raise ConanInvalidConfiguration("CCI has no thrift recipe (yet)") def requirements(self): + if self._with_thrift(): + self.requires("thrift/0.13.0") if self.options.with_backtrace: raise ConanInvalidConfiguration("CCI has no backtrace recipe (yet)") if self._with_protobuf(): @@ -445,6 +445,8 @@ def package_info(self): self.cpp_info.components["libarrow"].requires.append("uff8proc::uff8proc") if self._with_llvm(): self.cpp_info.components["libarrow"].requires.append("llvm::llvm") + if self._with_thrift(): + self.cpp_info.components["libarrow"].requires.append("thrift::thrift") if self.options.with_backtrace: self.cpp_info.components["libarrow"].requires.append("backtrace::backtrace") From b7e73303baab49fc0389ff8308b150a108bc8280 Mon Sep 17 00:00:00 2001 From: Anonymous Maarten Date: Fri, 14 Aug 2020 00:05:13 +0200 Subject: [PATCH 16/16] arrow: don't build on clang <= 3.9 --- recipes/arrow/all/conanfile.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/recipes/arrow/all/conanfile.py b/recipes/arrow/all/conanfile.py index 5e7dc6b2b0327..368fd39430bd0 100644 --- a/recipes/arrow/all/conanfile.py +++ b/recipes/arrow/all/conanfile.py @@ -101,6 +101,9 @@ def config_options(self): del self.options.fPIC def configure(self): + if self.settings.compiler == "clang" and self.settings.compiler.version <= tools.Version("3.9"): + raise ConanInvalidConfiguration("This recipe does not support this compiler version") + if self.options.shared: del self.options.fPIC if self.options.compute == False and not self._compute(True):