From 53a38ea14d751ebf5c84ef6c831ebdbbfc1208d1 Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Wed, 16 Sep 2020 14:28:34 +0100 Subject: [PATCH 1/8] Do not force prefixes, no longer required --- cmake/Modules/FindTileDB_EP.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmake/Modules/FindTileDB_EP.cmake b/cmake/Modules/FindTileDB_EP.cmake index 0133ba19..e4d6e7e8 100644 --- a/cmake/Modules/FindTileDB_EP.cmake +++ b/cmake/Modules/FindTileDB_EP.cmake @@ -44,8 +44,6 @@ if (NOT TILEDB_FOUND) GIT_REPOSITORY "${TILEDB_GIT_REPOSITORY}" GIT_TAG ${TILEDB_GIT_TAG} CMAKE_ARGS - -DCMAKE_INSTALL_PREFIX=${TILEDB_JNI_EP_INSTALL_PREFIX} - -DCMAKE_PREFIX_PATH=${TILEDB_JNI_EP_INSTALL_PREFIX} -DTILEDB_VERBOSE=${TILEDB_VERBOSE} -DTILEDB_S3=${TILEDB_S3} -DTILEDB_HDFS=${TILEDB_HDFS} From 3ecd3436297a3952137eb07f464fae6855fe835c Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Wed, 16 Sep 2020 14:28:52 +0100 Subject: [PATCH 2/8] Add and enable Azure support by default; helps build order In pursuing a manylinux2010 based build the openssl library lookups were failing when building cURL; likely because of misuse of the `--with-ssl` configure flag. As currently structured the configure generated cURL Makefile does not correctly add multilib paths to the linker search paths due to pkgconfig metadata not being used. Specifying `with-ssl` with a path creates this scenario. If openssl can find multilib [1] directories during the configure phase of the build it will use them. Currently, via cmake in this repository, TileDB will be built in such a way that openssl will be built after the `lib64` multilib directory has been created. TileDB-Py, which also uses a manylinux2010 build, has these same structural problems but includes Azure support by default. This changes the external dependency build order, openssl comes up first, the `lib64` directoriy is missing `LIBDIR` is set to `lib` and the rest of the build completes successfully. Not wanting to hack on the TileDB build itself the easiest solution was to add Azure support to TileDB-Java. It also has the nice side effect of including the support. 1. https://github.com/openssl/openssl/blob/OpenSSL_1_1_0h/Configurations/unix-Makefile.tmpl#L147-L155 --- build.gradle | 1 + cmake/Modules/FindTileDB_EP.cmake | 1 + gradle.properties | 1 + 3 files changed, 3 insertions(+) diff --git a/build.gradle b/build.gradle index 07447da6..a574b711 100644 --- a/build.gradle +++ b/build.gradle @@ -67,6 +67,7 @@ task cmakeTask(type: Exec) { "TILEDB_GIT_TAG", "TILEDB_VERBOSE", "TILEDB_S3", + "TILEDB_AZURE", "TILEDB_HDFS", "TILEDB_SERIALIZATION" ].each { v -> diff --git a/cmake/Modules/FindTileDB_EP.cmake b/cmake/Modules/FindTileDB_EP.cmake index e4d6e7e8..41709cf4 100644 --- a/cmake/Modules/FindTileDB_EP.cmake +++ b/cmake/Modules/FindTileDB_EP.cmake @@ -46,6 +46,7 @@ if (NOT TILEDB_FOUND) CMAKE_ARGS -DTILEDB_VERBOSE=${TILEDB_VERBOSE} -DTILEDB_S3=${TILEDB_S3} + -DTILEDB_AZURE=${TILEDB_AZURE} -DTILEDB_HDFS=${TILEDB_HDFS} -DTILEDB_SERIALIZATION=${TILEDB_SERIALIZATION} -DTILEDB_FORCE_ALL_DEPS=ON diff --git a/gradle.properties b/gradle.properties index 0a9fa64c..a80d7d2f 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,5 +2,6 @@ TILEDB_GIT_REPOSITORY=https://github.com/TileDB-Inc/TileDB TILEDB_GIT_TAG=2.0.8 TILEDB_VERBOSE=OFF TILEDB_S3=ON +TILEDB_AZURE=ON TILEDB_HDFS=OFF TILEDB_SERIALIZATION=OFF From a204acccb7ee8035c9802691be0eca1c348ffdfd Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Wed, 16 Sep 2020 16:32:09 +0100 Subject: [PATCH 3/8] Build release native artifacts using manylinux2010 --- ci/Dockerfile2010 | 10 ++++++++++ ci/build.sh | 8 ++++++++ ci/tiledb-java-linux_osx-release.yml | 11 ++++++----- 3 files changed, 24 insertions(+), 5 deletions(-) create mode 100644 ci/Dockerfile2010 create mode 100755 ci/build.sh diff --git a/ci/Dockerfile2010 b/ci/Dockerfile2010 new file mode 100644 index 00000000..5e1383d5 --- /dev/null +++ b/ci/Dockerfile2010 @@ -0,0 +1,10 @@ +# Dockerfile for lowest common denominator Linux native artifact build +# -------------------------------------------------------------------- +# Cribbed from TileDB-Py/misc/pypi_linux/Dockerfile2010 +FROM quay.io/pypa/manylinux2010_x86_64 + +RUN yum install -y java-1.8.0-openjdk-devel +RUN yum remove -y cmake + +ENV PATH /opt/python/cp38-cp38/bin:${PATH} +RUN pip install cmake==3.17.3 diff --git a/ci/build.sh b/ci/build.sh new file mode 100755 index 00000000..5f7cd51c --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -ex + +export TILEDB_PY_REPO="/opt/TileDB-Py" + +cd /TileDB-Java +./gradlew build diff --git a/ci/tiledb-java-linux_osx-release.yml b/ci/tiledb-java-linux_osx-release.yml index 56d78ea6..7f52bb62 100644 --- a/ci/tiledb-java-linux_osx-release.yml +++ b/ci/tiledb-java-linux_osx-release.yml @@ -12,14 +12,15 @@ steps: BUILD_BINARIESDIRECTORY=${BUILD_BINARIESDIRECTORY:-$BUILD_REPOSITORY_LOCALPATH/} - ./gradlew assemble - if [[ ( "$AGENT_OS" == "Linux" ) ]]; then - cp ./build/tiledb_jni/*.so ./build/install/lib/*.so $BUILD_BINARIESDIRECTORY + docker build -f ci/Dockerfile2010 . -t lib_builder_2010 + docker run -v $(pwd):/TileDB-Java -t lib_builder_2010 /TileDB-Java/ci/build.sh + cp ./build/tiledb_jni/*.so ./build/install/lib/*.so $BUILD_BINARIESDIRECTORY fi if [[ ( "$AGENT_OS" == "Darwin" ) ]]; then - cp ./build/tiledb_jni/*.dylib ./build/install/lib/*.dylib $BUILD_BINARIESDIRECTORY + ./gradlew assemble + cp ./build/tiledb_jni/*.dylib ./build/install/lib/*.dylib $BUILD_BINARIESDIRECTORY fi echo "Native Libs" @@ -65,4 +66,4 @@ steps: inputs: pathtoPublish: $(Build.ArtifactStagingDirectory)/tiledb-$(Agent.OS)-$(Build.SourceBranchName)-$(commitHash).tar.gz artifactName: libraries - condition: succeeded() \ No newline at end of file + condition: succeeded() From be8dcf31fa91dfd41f22f1fd2c665c54372b382d Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Wed, 16 Sep 2020 17:34:24 +0100 Subject: [PATCH 4/8] Revert "Do not force prefixes, no longer required" This reverts commit 53a38ea14d751ebf5c84ef6c831ebdbbfc1208d1. --- cmake/Modules/FindTileDB_EP.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/Modules/FindTileDB_EP.cmake b/cmake/Modules/FindTileDB_EP.cmake index 41709cf4..63fbf4c9 100644 --- a/cmake/Modules/FindTileDB_EP.cmake +++ b/cmake/Modules/FindTileDB_EP.cmake @@ -44,6 +44,8 @@ if (NOT TILEDB_FOUND) GIT_REPOSITORY "${TILEDB_GIT_REPOSITORY}" GIT_TAG ${TILEDB_GIT_TAG} CMAKE_ARGS + -DCMAKE_INSTALL_PREFIX=${TILEDB_JNI_EP_INSTALL_PREFIX} + -DCMAKE_PREFIX_PATH=${TILEDB_JNI_EP_INSTALL_PREFIX} -DTILEDB_VERBOSE=${TILEDB_VERBOSE} -DTILEDB_S3=${TILEDB_S3} -DTILEDB_AZURE=${TILEDB_AZURE} From 6a05a5707c61613e02fd9db54cf1b63831ff4094 Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Wed, 16 Sep 2020 18:02:06 +0100 Subject: [PATCH 5/8] Off by default as not currently working on Windows --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index a80d7d2f..1064a75a 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,6 +2,6 @@ TILEDB_GIT_REPOSITORY=https://github.com/TileDB-Inc/TileDB TILEDB_GIT_TAG=2.0.8 TILEDB_VERBOSE=OFF TILEDB_S3=ON -TILEDB_AZURE=ON +TILEDB_AZURE=OFF TILEDB_HDFS=OFF TILEDB_SERIALIZATION=OFF From bdc7e7381a063874874a0240bce57c496b6ce06b Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Wed, 16 Sep 2020 18:02:54 +0100 Subject: [PATCH 6/8] Remove unused environment variable --- ci/build.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/ci/build.sh b/ci/build.sh index 5f7cd51c..5a0848d6 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -2,7 +2,5 @@ set -ex -export TILEDB_PY_REPO="/opt/TileDB-Py" - cd /TileDB-Java ./gradlew build From 009ce18866e4f7940d22efafef67d6f8b13fdec8 Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Wed, 16 Sep 2020 18:05:28 +0100 Subject: [PATCH 7/8] Conditionally enable Azure support on Linux --- ci/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build.sh b/ci/build.sh index 5a0848d6..bdeb3540 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -3,4 +3,4 @@ set -ex cd /TileDB-Java -./gradlew build +./gradlew -PTILEDB_AZURE=ON build From 5a5f769dc8e151e7680bd3b5d5112f961490a7c9 Mon Sep 17 00:00:00 2001 From: Chris Allan Date: Wed, 16 Sep 2020 18:06:04 +0100 Subject: [PATCH 8/8] Conditionally enable Azure support on macOS --- ci/tiledb-java-linux_osx-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/tiledb-java-linux_osx-release.yml b/ci/tiledb-java-linux_osx-release.yml index 7f52bb62..84a28efe 100644 --- a/ci/tiledb-java-linux_osx-release.yml +++ b/ci/tiledb-java-linux_osx-release.yml @@ -19,7 +19,7 @@ steps: fi if [[ ( "$AGENT_OS" == "Darwin" ) ]]; then - ./gradlew assemble + ./gradlew -PTILEDB_AZURE=ON assemble cp ./build/tiledb_jni/*.dylib ./build/install/lib/*.dylib $BUILD_BINARIESDIRECTORY fi