Flat folder structure

apache · Nov 5, 2022 · 0c328c8 · 0c328c8
1 parent b7e591d
commit 0c328c8
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 89 deletions.
diff --git a/ci/scripts/java_build.sh b/ci/scripts/java_build.sh
@@ -93,7 +93,7 @@ fi
 
 if [ "${ARROW_PLASMA}" = "ON" ]; then
   pushd ${source_dir}/plasma
-  ${mvn} clean install
+  ${mvn} -Darrow.cpp.build.dir=${java_jni_dist_dir} clean install
   popd
 fi
 

diff --git a/cpp/Brewfile b/cpp/Brewfile
@@ -31,7 +31,6 @@ brew "grpc"
 brew "llvm@14"
 brew "lz4"
 brew "ninja"
-brew "numpy"
 brew "openssl@1.1"
 brew "protobuf"
 brew "python"

diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml
@@ -28,6 +28,8 @@ jobs:
       {{ macros.github_checkout_arrow()|indent }}
       {{ macros.github_install_archery()|indent }}
       - name: Build C++ libraries
+        env:
+        {{ macros.github_set_sccache_envvars()|indent(8) }}
         run: |
           archery docker run \
             -e ARROW_JAVA_BUILD=OFF \
@@ -47,89 +49,67 @@ jobs:
         run: archery docker push java-jni-manylinux-2014
     {% endif %}
 
-  build-cpp-macos-x86_64:
-    name: Build C++ libraries macOS x86_64
-    runs-on: macos-latest
+  build-cpp-macos:
+    {% set arch = '${{ matrix.platform.arch }}' %}
+    name: Build C++ libraries macOS {{ arch }}
+    runs-on: {{ '${{ matrix.platform.runs_on }}' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - { runs_on: ["macos-latest"], arch: "x86_64"}
+          - { runs_on: ["self-hosted", "macOS", "arm64", "devops-managed"], arch: "aarch_64" }
     env:
       MACOSX_DEPLOYMENT_TARGET: "10.13"
     steps:
       {{ macros.github_checkout_arrow()|indent }}
-      {{ macros.github_install_archery()|indent }}
-      - name: Install dependencies
-        run: |
-          brew install --overwrite git
-          brew bundle --file=arrow/cpp/Brewfile
-          # We want to link aws-sdk-cpp statically but Homebrew's
-          # aws-sdk-cpp provides only shared library. If we have
-          # Homebrew's aws-sdk-cpp, our build mix Homebrew's
-          # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
-          # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
-          brew uninstall aws-sdk-cpp
-      - name: Setup ccache
-        run: |
-          arrow/ci/scripts/ccache_setup.sh
-      - name: Build C++ libraries
-        run: |
-          set -e
-          arrow/ci/scripts/java_jni_macos_build.sh \
-            $GITHUB_WORKSPACE/arrow \
-            $GITHUB_WORKSPACE/arrow/cpp-build \
-            $GITHUB_WORKSPACE/arrow/java-dist
-      - name: Compress into single artifact to keep directory structure
-        run: |
-          mkdir -p arrow/java-dist/x86_64
-          mv arrow/java-dist/*.* arrow/java-dist/x86_64/
-          tar -cvzf arrow-shared-libs-macos-x86_64.tar.gz arrow/java-dist/
-      - name: Upload artifacts
-        uses: actions/upload-artifact@v2
+      - name: Set up Python
+        if: |
+          !contains(matrix.platform.runs_on, 'self-hosted')
+        uses: actions/setup-python@v4
         with:
-          name: macos-shared-lib-x86_64
-          path: arrow-shared-libs-macos-x86_64.tar.gz
-
-  build-cpp-macos-arm64:
-    name: Build C++ libraries macOS arm64
-    runs-on: ["self-hosted", "macOS", "arm64"]
-    env:
-      MACOSX_DEPLOYMENT_TARGET: "10.13"
-    steps:
-      {{ macros.github_checkout_arrow()|indent }}
+          cache: 'pip'
+          python-version: 3.8
+      - name: Install Archery
+        shell: bash
+        run: pip install -e arrow/dev/archery[all]
       - name: Install dependencies
         run: |
-          brew install --overwrite git
           brew bundle --file=arrow/cpp/Brewfile
           # We want to link aws-sdk-cpp statically but Homebrew's
           # aws-sdk-cpp provides only shared library. If we have
           # Homebrew's aws-sdk-cpp, our build mix Homebrew's
           # aws-sdk-cpp and bundled aws-sdk-cpp. We uninstall Homebrew's
           # aws-sdk-cpp to ensure using only bundled aws-sdk-cpp.
           brew uninstall aws-sdk-cpp
-      - name: Setup ccache
-        run: |
-          arrow/ci/scripts/ccache_setup.sh
+          brew install openjdk@11
+          brew install sccache
       - name: Build C++ libraries
+        env:
+        {{ macros.github_set_sccache_envvars()|indent(8) }}
         run: |
           set -e
+          # make brew Java available to CMake
+          if [ "{{ arch }}" = "aarch_64" ]; then
+            export JAVA_HOME=$(brew --prefix openjdk@11)/libexec/openjdk.jdk/Contents/Home
+          fi
           arrow/ci/scripts/java_jni_macos_build.sh \
             $GITHUB_WORKSPACE/arrow \
             $GITHUB_WORKSPACE/arrow/cpp-build \
             $GITHUB_WORKSPACE/arrow/java-dist
       - name: Compress into single artifact to keep directory structure
-        run: |
-          mkdir -p arrow/java-dist/aarch_64
-          mv arrow/java-dist/*.* arrow/java-dist/aarch_64/
-          tar -cvzf arrow-shared-libs-macos-arm64.tar.gz arrow/java-dist/
+        run: tar -cvzf arrow-shared-libs-macos-{{ arch }}.tar.gz arrow/java-dist/
       - name: Upload artifacts
         uses: actions/upload-artifact@v2
         with:
-          name: macos-shared-lib-arm64
-          path: arrow-shared-libs-macos-arm64.tar.gz
+          name: macos-shared-lib-{{ arch }}
+          path: arrow-shared-libs-macos-{{ arch }}.tar.gz
 
   build-cpp-windows:
     name: Build C++ libraries Windows
     runs-on: windows-2019
     steps:
       {{ macros.github_checkout_arrow()|indent }}
-      {{ macros.github_install_archery()|indent }}
       - name: Set up Java
         uses: actions/setup-java@v3
         with:
@@ -138,8 +118,13 @@ jobs:
       - name: Download Timezone Database
         shell: bash
         run: arrow/ci/scripts/download_tz_database.sh
+      - name: Install sccache
+        shell: bash
+        run: arrow/ci/scripts/install_sccache.sh pc-windows-msvc $(pwd)/sccache
       - name: Build C++ libraries
         shell: cmd
+        env:
+        {{ macros.github_set_sccache_envvars()|indent(8) }}
         run: |
           call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" x64
           REM For ORC
@@ -159,52 +144,46 @@ jobs:
     runs-on: macos-latest
     needs:
       - build-cpp-ubuntu
-      - build-cpp-macos-x86_64
-      - build-cpp-macos-arm64
+      - build-cpp-macos
       - build-cpp-windows
     steps:
       {{ macros.github_checkout_arrow(fetch_depth=0)|indent }}
-      - name: Download Linux C++ Libraries
-        uses: actions/download-artifact@v2
-        with:
-          name: ubuntu-shared-lib
-      - name: Download macOS x86_64 C++ libraries
-        uses: actions/download-artifact@v2
-        with:
-          name: macos-shared-lib-x86_64
-      - name: Download Windows C++ libraries
-        uses: actions/download-artifact@v2
+      - name: Download Libraries
+        uses: actions/download-artifact@v3
         with:
-          name: windows-shared-lib
-      - name: Download macOS arm64 C++ libraries
-        uses: actions/download-artifact@v2
-        with:
-          name: macos-shared-lib-arm64
-      - name: Descompress artifacts
+          path: artifacts
+      - name: Decompress artifacts
         run: |
+          mv artifacts/*/*.tar.gz .
           tar -xvzf arrow-shared-libs-linux.tar.gz
           tar -xvzf arrow-shared-libs-macos-x86_64.tar.gz
-          tar -xvzf arrow-shared-libs-macos-arm64.tar.gz
+          tar -xvzf arrow-shared-libs-macos-aarch_64.tar.gz
           tar -xvzf arrow-shared-libs-windows.tar.gz
       - name: Test that shared libraries exist
         run: |
           set -x
+
           test -f arrow/java-dist/libarrow_cdata_jni.so
-          test -f arrow/java-dist/arrow_cdata_jni.dll
-          test -f arrow/java-dist/x86_64/libarrow_dataset_jni.dylib
-          test -f arrow/java-dist/aarch_64/libarrow_dataset_jni.dylib
           test -f arrow/java-dist/libarrow_dataset_jni.so
-          test -f arrow/java-dist/arrow_dataset_jni.dll
-          test -f arrow/java-dist/x86_64/libarrow_orc_jni.dylib
-          test -f arrow/java-dist/aarch_64/libarrow_orc_jni.dylib
           test -f arrow/java-dist/libarrow_orc_jni.so
-          test -f arrow/java-dist/arrow_orc_jni.dll
-          test -f arrow/java-dist/x86_64/libgandiva_jni.dylib
-          test -f arrow/java-dist/aarch_64/libgandiva_jni.dylib
           test -f arrow/java-dist/libgandiva_jni.so
-          test -f arrow/java-dist/x86_64/libplasma_java.dylib
-          test -f arrow/java-dist/aarch_64/libplasma_java.dylib
           test -f arrow/java-dist/libplasma_java.so
+
+          test -f arrow/java-dist/libarrow_cdata_jni.dylib
+          test -f arrow/java-dist/libarrow_dataset_jni.dylib
+          test -f arrow/java-dist/libarrow_orc_jni.dylib
+          test -f arrow/java-dist/libgandiva_jni.dylib
+          test -f arrow/java-dist/libplasma_java.dylib
+
+          test -f arrow/java-dist/libarrow_cdata_jni.dylib
+          test -f arrow/java-dist/libarrow_dataset_jni.dylib
+          test -f arrow/java-dist/libarrow_orc_jni.dylib
+          test -f arrow/java-dist/libgandiva_jni.dylib
+          test -f arrow/java-dist/libplasma_java.dylib
+
+          test -f arrow/java-dist/arrow_cdata_jni.dll
+          test -f arrow/java-dist/arrow_dataset_jni.dll
+          test -f arrow/java-dist/arrow_orc_jni.dll
       - name: Build bundled jar
         run: |
           set -e

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1048,10 +1048,10 @@ services:
       - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
     command:
       ["pip install -e /arrow/dev/archery && \
-        /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist/x86_64 && \
+        /arrow/ci/scripts/java_jni_manylinux_build.sh /arrow /build /arrow/java-dist && \
         source /opt/rh/rh-maven35/enable && \
-        /arrow/ci/scripts/java_build.sh /arrow /build /arrow/java-dist/x86_64 && \
-        /arrow/ci/scripts/java_test.sh /arrow /build /arrow/java-dist/x86_64"]
+        /arrow/ci/scripts/java_build.sh /arrow /build /arrow/java-dist && \
+        /arrow/ci/scripts/java_test.sh /arrow /build /arrow/java-dist"]
 
   ##############################  Integration #################################
 

diff --git a/java/plasma/pom.xml b/java/plasma/pom.xml
@@ -20,7 +20,19 @@
     <name>Arrow Plasma Client</name>
     <description>(DEPRECATED) Java client for the Plasma object store.
         This is deprecated since 10.0.0. This will be removed from 12.0.0 or so.</description>
+    <properties>
+        <arrow.cpp.build.dir>../../../cpp/release-build</arrow.cpp.build.dir>
+    </properties>
     <build>
+        <resources>
+            <resource>
+                <directory>${arrow.cpp.build.dir}</directory>
+                <includes>
+                    <include>**/*plasma_java.*</include>
+                </includes>
+            </resource>
+        </resources>
+
         <plugins>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>

diff --git a/java/pom.xml b/java/pom.xml
@@ -1058,7 +1058,7 @@
                     -DCMAKE_BUILD_TYPE=Release
                     -DCMAKE_INSTALL_LIBDIR=lib
                     -DCMAKE_INSTALL_PREFIX=${arrow.dataset.jni.dist.dir}
-                    -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist/${os.detected.arch}
+                    -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist
                   </commandlineArgs>
                   <workingDirectory>../</workingDirectory>
                 </configuration>
@@ -1175,7 +1175,7 @@
                     -DCMAKE_BUILD_TYPE=Release
                     -DCMAKE_INSTALL_LIBDIR=lib
                     -DCMAKE_INSTALL_PREFIX=${arrow.dataset.jni.dist.dir}
-                    -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist/${os.detected.arch}
+                    -DCMAKE_PREFIX_PATH=${project.basedir}/../java-dist
                   </commandlineArgs>
                   <workingDirectory>../</workingDirectory>
                 </configuration>