Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ARROW-17795: [C++][R] Add missing PKG_CONFIG_PATH to use system zstd #14202

Merged
merged 10 commits into from
Sep 27, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cpp/cmake_modules/FindzstdAlt.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -109,4 +109,6 @@ if(zstdAlt_FOUND)
set_target_properties(${zstd_TARGET}
PROPERTIES IMPORTED_LOCATION "${ZSTD_LIB}"
INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}")
message(STATUS "Zstandard library: ${ZSTD_LIB}")
message(STATUS "Zstandard include directory: ${ZSTD_INCLUDE_DIR}")
endif()
17 changes: 11 additions & 6 deletions cpp/cmake_modules/ThirdpartyToolchain.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,12 @@ macro(resolve_dependency DEPENDENCY_NAME)
NO_CMAKE_ENVIRONMENT_PATH
QUIET)
if(${${ARG_PC_PACKAGE_NAME}_PC_FOUND})
message(STATUS "Using pkg-config package for ${ARG_PC_PACKAGE_NAME} for static link"
)
string(APPEND ARROW_PC_REQUIRES_PRIVATE " ${ARG_PC_PACKAGE_NAME}")
else()
message(STATUS "pkg-config package for ${ARG_PC_PACKAGE_NAME} for static link isn't found"
)
endif()
endforeach()
endif()
Expand Down Expand Up @@ -2421,14 +2426,14 @@ if(ARROW_WITH_ZSTD)
set(ARROW_ZSTD_LIBZSTD zstd::libzstd_static)
else()
if(ARROW_ZSTD_USE_SHARED)
if(TARGET zstd::libzstd_shared)
set(ARROW_ZSTD_LIBZSTD zstd::libzstd_shared)
endif()
set(ARROW_ZSTD_LIBZSTD zstd::libzstd_shared)
else()
if(TARGET zstd::libzstd_static)
set(ARROW_ZSTD_LIBZSTD zstd::libzstd_static)
endif()
set(ARROW_ZSTD_LIBZSTD zstd::libzstd_static)
endif()
if(NOT TARGET ${ARROW_ZSTD_LIBZSTD})
message(FATAL_ERROR "Zstandard target doesn't exist: ${ARROW_ZSTD_LIBZSTD}")
endif()
message(STATUS "Found Zstandard: ${ARROW_ZSTD_LIBZSTD}")
endif()
endif()

Expand Down
5 changes: 5 additions & 0 deletions dev/tasks/r/github.linux.arrow.version.back.compat.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ jobs:
{{ macros.github_checkout_arrow()|indent }}

- uses: r-lib/actions/setup-r@v1
- name: Prepare PKG_CONFIG_PATH for Homebrew
run: |
# zstd is installed by Homebrew on GitHub Actions.
echo "PKG_CONFIG_PATH=$(brew --prefix)/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" \
>> "$GITHUB_ENV"
- name: Install dependencies
run: |
install.packages(c("remotes", "glue", "sys"))
Expand Down
5 changes: 5 additions & 0 deletions dev/tasks/r/github.linux.offline.build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,11 @@ jobs:
run: |
sudo apt-get update
sudo apt install libcurl4-openssl-dev libssl-dev
- name: Prepare PKG_CONFIG_PATH for Homebrew
run: |
# zstd is installed by Homebrew on GitHub Actions.
echo "PKG_CONFIG_PATH=$(brew --prefix)/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" \
>> "$GITHUB_ENV"
- name: Install dependencies
run: |
install.packages(c("remotes", "glue", "sys"))
Expand Down
3 changes: 3 additions & 0 deletions dev/tasks/r/github.macos-linux.local.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ jobs:
cd arrow/r
sudo apt-get update
sudo apt install libcurl4-openssl-dev libssl-dev
# zstd is installed by Homebrew on GitHub Actions.
echo "PKG_CONFIG_PATH=$(brew --prefix)/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" \
>> "$GITHUB_ENV"
if: contains(matrix.os, 'ubuntu')
- uses: r-lib/actions/setup-r@v1
- name: Install dependencies
Expand Down
6 changes: 6 additions & 0 deletions dev/tasks/r/github.packages.yml
Original file line number Diff line number Diff line change
Expand Up @@ -310,6 +310,11 @@ jobs:
https://raw.githubusercontent.com/{{ arrow.github_repo }}/{{ arrow.head }}/ci/scripts/install_sccache.sh | \
bash -s unknown-linux-musl /usr/local/bin
- run: sudo apt update && sudo apt install libcurl4-openssl-dev
- name: Prepare PKG_CONFIG_PATH for Homebrew
run: |
# zstd is installed by Homebrew on GitHub Actions.
echo "PKG_CONFIG_PATH=$(brew --prefix)/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" \
>> "$GITHUB_ENV"
- name: Install arrow from nightly repo
env:
# Test source build so be sure not to download a binary
Expand All @@ -324,6 +329,7 @@ jobs:
env:
LIBARROW_BINARY: "FALSE"
ARROW_R_DEV: "TRUE"
CMAKE_FIND_DEBUG_MODE: "ON"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we remove this before merge?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I want to keep this to debug a problem in the future.
This step is executed only when the above build step is failed. Is it accepted?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 makes sense!

{{ macros.github_set_sccache_envvars()|indent(8) }}
shell: Rscript {0}
run: |
Expand Down
4 changes: 2 additions & 2 deletions r/configure
Original file line number Diff line number Diff line change
Expand Up @@ -184,8 +184,8 @@ else

# Use pkg-config to do static linking of libarrow's dependencies
if [ "$ARROW_DEPENDENCY_SOURCE" = "AUTO" ] || [ "$ARROW_DEPENDENCY_SOURCE" = "SYSTEM" ]; then
PKG_LIBS="`PKG_CONFIG_PATH=${LIB_DIR}/pkgconfig pkg-config --libs-only-l --static --silence-errors ${PKG_CONFIG_NAME}`"
PKG_LIBS="$PKG_LIBS `PKG_CONFIG_PATH=${LIB_DIR}/pkgconfig pkg-config --libs-only-other --static --silence-errors ${PKG_CONFIG_NAME}`"
export PKG_CONFIG_PATH=${LIB_DIR}/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}
PKG_LIBS="`pkg-config --libs --static --silence-errors ${PKG_CONFIG_NAME}`"
Comment on lines +187 to +188
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These changes are not required for configure.win because we use the mysys stuff anyway right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I didn't see configure.win but it seems that configure.win doesn't use pkg-config on release build. configure.win uses pkg-config on dev build but it already uses pkg-config --libs ....

fi

# When using brew's openssl it is not bundled and it is not on the system
Expand Down
2 changes: 2 additions & 0 deletions r/inst/build_arrow_static.sh
Original file line number Diff line number Diff line change
Expand Up @@ -75,12 +75,14 @@ ${CMAKE} -DARROW_BOOST_USE_SHARED=OFF \
-DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD:-$ARROW_DEFAULT_PARAM} \
-DARROW_VERBOSE_THIRDPARTY_BUILD=${ARROW_VERBOSE_THIRDPARTY_BUILD:-OFF} \
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-Release} \
-DCMAKE_FIND_DEBUG_MODE=${CMAKE_FIND_DEBUG_MODE:-OFF} \
-DCMAKE_INSTALL_LIBDIR=lib \
-DCMAKE_INSTALL_PREFIX=${DEST_DIR} \
-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON \
-DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON \
-DCMAKE_UNITY_BUILD=${CMAKE_UNITY_BUILD:-OFF} \
-Dxsimd_SOURCE=${xsimd_SOURCE:-} \
-Dzstd_SOURCE=${zstd_SOURCE:-} \
${EXTRA_CMAKE_FLAGS} \
-G ${CMAKE_GENERATOR:-"Unix Makefiles"} \
${SOURCE_DIR}
Expand Down