From 47b38cd6dbcf4abc8ac131f12944b6f064ba7372 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 31 May 2017 13:51:40 -0400 Subject: [PATCH 1/3] Clean up after libparquet/libparquet_arrow consolidation Change-Id: I5af8c04d37777a0d80c4d4c7d71028038d9e1e1a --- python/CMakeLists.txt | 27 ++--------------- python/cmake_modules/FindParquet.cmake | 40 +------------------------- 2 files changed, 3 insertions(+), 64 deletions(-) diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt index b02f23bfff9e..f35326a642f5 100644 --- a/python/CMakeLists.txt +++ b/python/CMakeLists.txt @@ -290,7 +290,7 @@ if (PYARROW_BUILD_PARQUET) ## Parquet find_package(Parquet) - if(NOT (PARQUET_FOUND AND PARQUET_ARROW_FOUND)) + if(NOT PARQUET_FOUND) message(FATAL_ERROR "Unable to locate Parquet libraries") endif() include_directories(SYSTEM ${PARQUET_INCLUDE_DIR}) @@ -298,14 +298,10 @@ if (PYARROW_BUILD_PARQUET) if (PYARROW_BUNDLE_ARROW_CPP) get_filename_component(PARQUET_LIBRARY_DIR ${PARQUET_SHARED_LIB} DIRECTORY) get_filename_component(PARQUET_LIBRARY_NAME ${PARQUET_SHARED_LIB} NAME_WE) - get_filename_component(PARQUET_ARROW_LIBRARY_NAME ${PARQUET_ARROW_SHARED_LIB} NAME_WE) message(STATUS "Configure start") configure_file(${PARQUET_SHARED_LIB} ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} COPYONLY) - configure_file(${PARQUET_ARROW_SHARED_LIB} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/${PARQUET_ARROW_LIBRARY_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} - COPYONLY) if (APPLE) configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} @@ -313,12 +309,6 @@ if (PYARROW_BUILD_PARQUET) configure_file(${PARQUET_LIBRARY_DIR}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} COPYONLY) - configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_ABI_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - COPYONLY) - configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow.${PARQUET_SO_VERSION}${CMAKE_SHARED_LIBRARY_SUFFIX} - COPYONLY) elseif (NOT MSVC) configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} @@ -326,34 +316,21 @@ if (PYARROW_BUILD_PARQUET) configure_file(${PARQUET_LIBRARY_DIR}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} COPYONLY) - configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_ABI_VERSION} - COPYONLY) - configure_file(${PARQUET_LIBRARY_DIR}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} - ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}.${PARQUET_SO_VERSION} - COPYONLY) endif() message(STATUS "Configure end") #SET(PARQUET_SHARED_LIB # ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet${CMAKE_SHARED_LIBRARY_SUFFIX}) - #SET(PARQUET_ARROW_SHARED_LIB - # ${BUILD_OUTPUT_ROOT_DIRECTORY}/libparquet_arrow${CMAKE_SHARED_LIBRARY_SUFFIX}) endif() if (MSVC) ADD_THIRDPARTY_LIB(parquet SHARED_LIB ${PARQUET_SHARED_IMP_LIB}) - ADD_THIRDPARTY_LIB(parquet_arrow - SHARED_LIB ${PARQUET_ARROW_SHARED_IMP_LIB}) else() ADD_THIRDPARTY_LIB(parquet SHARED_LIB ${PARQUET_SHARED_LIB}) - ADD_THIRDPARTY_LIB(parquet_arrow - SHARED_LIB ${PARQUET_ARROW_SHARED_LIB}) endif() set(LINK_LIBS ${LINK_LIBS} - parquet_shared - parquet_arrow_shared) + parquet_shared) set(CYTHON_EXTENSIONS ${CYTHON_EXTENSIONS} _parquet) diff --git a/python/cmake_modules/FindParquet.cmake b/python/cmake_modules/FindParquet.cmake index ef3c645f7682..88dca2ed6464 100644 --- a/python/cmake_modules/FindParquet.cmake +++ b/python/cmake_modules/FindParquet.cmake @@ -41,11 +41,7 @@ if (MSVC) PATHS ${PARQUET_HOME} NO_DEFAULT_PATH PATH_SUFFIXES "bin" ) - find_library(PARQUET_ARROW_SHARED_LIBRARIES NAMES parquet_arrow - PATHS ${PARQUET_HOME} NO_DEFAULT_PATH - PATH_SUFFIXES "bin" ) get_filename_component(PARQUET_SHARED_LIBS ${PARQUET_SHARED_LIBRARIES} PATH ) - get_filename_component(PARQUET_ARROW_SHARED_LIBS ${PARQUET_ARROW_SHARED_LIBRARIES} PATH ) endif () if(PARQUET_HOME) @@ -63,9 +59,6 @@ if(PARQUET_HOME) find_library(PARQUET_LIBRARIES NAMES parquet PATHS ${PARQUET_HOME} NO_DEFAULT_PATH PATH_SUFFIXES "lib") - find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow - PATHS ${PARQUET_HOME} NO_DEFAULT_PATH - PATH_SUFFIXES "lib") get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH ) else() pkg_check_modules(PARQUET parquet) @@ -80,13 +73,9 @@ else() message(STATUS "Searching for parquet libs in: ${PARQUET_SEARCH_LIB_PATH}") find_library(PARQUET_LIBRARIES NAMES parquet PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH) - find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow - PATHS ${PARQUET_SEARCH_LIB_PATH} NO_DEFAULT_PATH) - message(STATUS "${PARQUET_ARROW_LIBRARIES}") else() find_path(PARQUET_INCLUDE_DIR NAMES parquet/api/reader.h ) find_library(PARQUET_LIBRARIES NAMES parquet) - find_library(PARQUET_ARROW_LIBRARIES NAMES parquet_arrow) get_filename_component(PARQUET_LIBS ${PARQUET_LIBRARIES} PATH ) endif() endif() @@ -106,28 +95,9 @@ else () set(PARQUET_FOUND FALSE) endif () -if (PARQUET_INCLUDE_DIR AND PARQUET_ARROW_LIBRARIES) - set(PARQUET_ARROW_FOUND TRUE) - get_filename_component(PARQUET_ARROW_LIBS ${PARQUET_ARROW_LIBRARIES} PATH) - set(PARQUET_ARROW_LIB_NAME parquet_arrow) - if (MSVC) - set(PARQUET_ARROW_STATIC_LIB "${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}${PARQUET_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(PARQUET_ARROW_SHARED_LIB "${PARQUET_ARROW_SHARED_LIBS}/${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}") - set(PARQUET_ARROW_SHARED_IMP_LIB "${PARQUET_ARROW_LIBS}/${PARQUET_ARROW_LIB_NAME}.lib") - else() - set(PARQUET_ARROW_STATIC_LIB - ${PARQUET_ARROW_LIBS}/${CMAKE_STATIC_LIBRARY_PREFIX}${PARQUET_ARROW_LIB_NAME}.a) - set(PARQUET_ARROW_SHARED_LIB - ${PARQUET_ARROW_LIBS}/${CMAKE_SHARED_LIBRARY_PREFIX}${PARQUET_ARROW_LIB_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) - endif() -else () - set(PARQUET_ARROW_FOUND FALSE) -endif () - -if (PARQUET_FOUND AND PARQUET_ARROW_FOUND) +if (PARQUET_FOUND) if (NOT Parquet_FIND_QUIETLY) message(STATUS "Found the Parquet library: ${PARQUET_LIBRARIES}") - message(STATUS "Found the Parquet Arrow library: ${PARQUET_ARROW_LIBS}") endif () else () if (NOT Parquet_FIND_QUIETLY) @@ -135,9 +105,6 @@ else () set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Could not find the parquet library.") endif() - if (NOT PARQUET_ARROW_FOUND) - set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Could not find the parquet_arrow library. Did you build with -DPARQUET_ARROW=on?") - endif() set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} Looked in ") if ( _parquet_roots ) set(PARQUET_ERR_MSG "${PARQUET_ERR_MSG} in ${_parquet_roots}.") @@ -159,9 +126,4 @@ mark_as_advanced( PARQUET_LIBRARIES PARQUET_STATIC_LIB PARQUET_SHARED_LIB - - PARQUET_ARROW_FOUND - PARQUET_ARROW_LIBS - PARQUET_ARROW_STATIC_LIB - PARQUET_ARROW_SHARED_LIB ) From 9bd6c365efa3f17d54b5c2e2b2aa3e391c51d6b1 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 31 May 2017 13:53:18 -0400 Subject: [PATCH 2/3] Remove PARQUET_ARROW CMake flag in build fileS Change-Id: I4d567cc92778fff4590c6fa45b2ea5d3cc99a39e --- ci/travis_script_python.sh | 1 - cpp/doc/Parquet.md | 2 +- python/manylinux1/Dockerfile-x86_64 | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ci/travis_script_python.sh b/ci/travis_script_python.sh index ce5f7ec506b7..c3735ccadac3 100755 --- a/ci/travis_script_python.sh +++ b/ci/travis_script_python.sh @@ -49,7 +49,6 @@ build_parquet_cpp() { cmake \ -DCMAKE_BUILD_TYPE=debug \ -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME \ - -DPARQUET_ARROW=on \ -DPARQUET_BUILD_BENCHMARKS=off \ -DPARQUET_BUILD_EXECUTABLES=off \ -DPARQUET_ZLIB_VENDORED=off \ diff --git a/cpp/doc/Parquet.md b/cpp/doc/Parquet.md index 4985dd3b0bc2..ce2961ab26a0 100644 --- a/cpp/doc/Parquet.md +++ b/cpp/doc/Parquet.md @@ -24,7 +24,7 @@ export ARROW_HOME=$HOME/local git clone https://github.com/apache/parquet-cpp.git cd parquet-cpp -cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME -DPARQUET_ARROW=on +cmake -DCMAKE_INSTALL_PREFIX=$PARQUET_HOME make -j4 make install ``` diff --git a/python/manylinux1/Dockerfile-x86_64 b/python/manylinux1/Dockerfile-x86_64 index 363308b616e7..11e66f7d0cdb 100644 --- a/python/manylinux1/Dockerfile-x86_64 +++ b/python/manylinux1/Dockerfile-x86_64 @@ -21,5 +21,5 @@ RUN make -j5 install WORKDIR / RUN git clone https://github.com/apache/parquet-cpp.git WORKDIR /parquet-cpp -RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_ARROW=ON -DPARQUET_BOOST_USE_SHARED=OFF . +RUN ARROW_HOME=/arrow-dist cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/arrow-dist -DPARQUET_BUILD_TESTS=OFF -DPARQUET_BOOST_USE_SHARED=OFF . RUN make -j5 install From 4bcf580ebe2be802caa21bf072bcce99adadb26a Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Wed, 31 May 2017 17:21:22 -0400 Subject: [PATCH 3/3] Remove move_lib for parquet_arrow Change-Id: I4b96a11a1a3352e08737ebf5a2b475cf569a60bf --- python/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 7e612c9e274f..b64e2591298a 100644 --- a/python/setup.py +++ b/python/setup.py @@ -248,7 +248,6 @@ def move_lib(lib_name): move_lib("arrow_jemalloc") if self.with_parquet: move_lib("parquet") - move_lib("parquet_arrow") # Move the built C-extension to the place expected by the Python build self._found_names = []