ARROW-8335: [Release] Add crossbow jobs to run release verification

Closes #6836 from nealrichardson/verify-rc Lead-authored-by: Neal Richardson <neal.p.richardson@gmail.com> Co-authored-by: Krisztián Szűcs <szucs.krisztian@gmail.com> Signed-off-by: Wes McKinney <wesm+git@apache.org>
apache · Apr 10, 2020 · e570db9 · e570db9
1 parent 2914899
commit e570db9
Show file tree

Hide file tree

Showing 7 changed files with 388 additions and 23 deletions.
diff --git a/dev/release/verify-release-candidate-wheels.bat b/dev/release/verify-release-candidate-wheels.bat
@@ -49,7 +49,9 @@ EXIT /B %ERRORLEVEL%
 
 :error
 call deactivate
-goto done
+cd %_CURRENT_DIR%
+
+EXIT /B 1
 
 @rem a batch function to verify a single wheel
 :verify_wheel

diff --git a/dev/release/verify-release-candidate.bat b/dev/release/verify-release-candidate.bat
@@ -33,7 +33,7 @@ set ARROW_SOURCE=%_VERIFICATION_DIR%\apache-arrow-%1
 set INSTALL_DIR=%_VERIFICATION_DIR%\install
 
 @rem Requires GNU Wget for Windows
-wget --no-check-certificate -O %_TARBALL% %_DIST_URL%/apache-arrow-%1-rc%2/%_TARBALL% || exit /B
+wget --no-check-certificate -O %_TARBALL% %_DIST_URL%/apache-arrow-%1-rc%2/%_TARBALL% || exit /B 1
 
 tar xf %_TARBALL% -C %_VERIFICATION_DIR_UNIX%
 
@@ -43,19 +43,19 @@ set PYTHON=3.6
 @rem script execution
 call conda create -p %_VERIFICATION_CONDA_ENV% ^
     --no-shortcuts -f -q -y python=%PYTHON% ^
-    || exit /B
+    || exit /B 1
 
-call activate %_VERIFICATION_CONDA_ENV% || exit /B
+call activate %_VERIFICATION_CONDA_ENV% || exit /B 1
 
 call conda install -y ^
      --no-shortcuts ^
      python=3.7 ^
      git ^
      --file=ci\conda_env_cpp.yml ^
      --file=ci\conda_env_python.yml ^
-     -c conda-forge || exit /B
+     -c conda-forge || exit /B 1
 
-set GENERATOR=Visual Studio 14 2015 Win64
+set GENERATOR=Visual Studio 15 2017 Win64
 set CONFIGURATION=release
 
 pushd %ARROW_SOURCE%
@@ -99,8 +99,8 @@ cmake --build . --target INSTALL --config Release
 @rem NOTE(wesm): Building googletest is flaky for me with ninja. Building it
 @rem first fixes the problem
 
-@rem ninja googletest_ep || exit /B
-@rem ninja install || exit /B
+@rem ninja googletest_ep || exit /B 1
+@rem ninja install || exit /B 1
 
 @rem Get testing datasets for Parquet unit tests
 git clone https://github.com/apache/parquet-testing.git %_VERIFICATION_DIR%\parquet-testing
@@ -112,7 +112,7 @@ set ARROW_TEST_DATA=%_VERIFICATION_DIR%\arrow-testing\data
 @rem Needed so python-test.exe works
 set PYTHONPATH=%CONDA_PREFIX%\Lib;%CONDA_PREFIX%\Lib\site-packages;%CONDA_PREFIX%\python35.zip;%CONDA_PREFIX%\DLLs;%CONDA_PREFIX%;%PYTHONPATH%
 
-ctest -VV  || exit /B
+ctest -VV  || exit /B 1
 popd
 
 @rem Build and import pyarrow
@@ -122,8 +122,8 @@ set PYARROW_CMAKE_GENERATOR=%GENERATOR%
 set PYARROW_WITH_FLIGHT=1
 set PYARROW_WITH_PARQUET=1
 set PYARROW_WITH_DATASET=1
-python setup.py build_ext --inplace --bundle-arrow-cpp bdist_wheel  || exit /B
-py.test pyarrow -v -s --parquet || exit /B
+python setup.py build_ext --inplace --bundle-arrow-cpp bdist_wheel  || exit /B 1
+py.test pyarrow -v -s --parquet || exit /B 1
 
 popd
 

diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
@@ -277,6 +277,7 @@ ${ARROW_CMAKE_OPTIONS:-}
 -DARROW_PARQUET=ON
 -DARROW_DATASET=ON
 -DPARQUET_REQUIRE_ENCRYPTION=ON
+-DARROW_VERBOSE_THIRDPARTY_BUILD=ON
 -DARROW_WITH_BZ2=ON
 -DARROW_WITH_ZLIB=ON
 -DARROW_WITH_ZSTD=ON
@@ -414,12 +415,12 @@ test_glib() {
 test_js() {
   pushd js
 
-  export NVM_DIR="`pwd`/.nvm"
-  mkdir -p $NVM_DIR
-  curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
-  [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
-
-  nvm install node
+  # export NVM_DIR="`pwd`/.nvm"
+  # mkdir -p $NVM_DIR
+  # curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
+  # [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh"
+  #
+  # nvm install node
 
   npm install
   # clean, lint, and build JS source
@@ -648,10 +649,8 @@ test_linux_wheels() {
       pip install python-rc/${VERSION}-rc${RC_NUMBER}/pyarrow-${VERSION}-cp${py_arch//[mu.]/}-cp${py_arch//./}-manylinux${ml_spec}_x86_64.whl
       check_python_imports py_arch
 
-      # install test requirements
+      # install test requirements and execute the tests
       pip install -r ${ARROW_DIR}/python/requirements-test.txt
-
-      # execute the python unit tests
       pytest --pyargs pyarrow
     done
 
@@ -681,10 +680,8 @@ test_macos_wheels() {
     pip install python-rc/${VERSION}-rc${RC_NUMBER}/pyarrow-${VERSION}-cp${py_arch//[m.]/}-cp${py_arch//./}-${macos_suffix}.whl
     check_python_imports py_arch
 
-    # install test requirements
+    # install test requirements and execute the tests
     pip install -r ${ARROW_DIR}/python/requirements-test.txt
-
-    # execute the python unit tests
     pytest --pyargs pyarrow
 
     conda deactivate

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -73,6 +73,21 @@ groups:
     - test-*spark*
     - test-*turbodbc*
 
+  verify-rc:
+    - verify-rc-*
+
+  verify-rc-wheels:
+    - verify-rc-wheels-*
+
+  verify-rc-source:
+    - verify-rc-source-*
+
+  verify-rc-source-macos:
+    - verify-rc-source-macos-*
+
+  verify-rc-source-linux:
+    - verify-rc-source-linux-*
+
   ######################## Tasks to run regularly #############################
 
   nightly:
@@ -1401,6 +1416,189 @@ tasks:
     artifacts:
       - arrow-gandiva-{no_rc_version}-SNAPSHOT.jar
 
+  ########################### Release verification ############################
+
+  verify-rc-binaries:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "ubuntu"
+      artifact: "binaries"
+      flag: ""
+
+  verify-rc-wheels-linux:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "ubuntu"
+      artifact: "wheels"
+      flag: ""
+
+  verify-rc-wheels-macos:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "macOS"
+      artifact: "wheels"
+      flag: ""
+
+  verify-rc-source-macos-java:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "macOS"
+      artifact: "source"
+      flag: "TEST_JAVA=1"
+
+  verify-rc-source-macos-csharp:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "macOS"
+      artifact: "source"
+      flag: "TEST_CSHARP=1"
+
+  verify-rc-source-macos-ruby:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "macOS"
+      artifact: "source"
+      flag: "TEST_RUBY=1"
+
+  verify-rc-source-macos-python:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "macOS"
+      artifact: "source"
+      flag: "TEST_PYTHON=1"
+
+  verify-rc-source-macos-js:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "macOS"
+      artifact: "source"
+      flag: "TEST_JS=1"
+
+  verify-rc-source-macos-go:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "macOS"
+      artifact: "source"
+      flag: "TEST_GO=1"
+
+  verify-rc-source-macos-rust:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "macOS"
+      artifact: "source"
+      flag: "TEST_RUST=1"
+
+  verify-rc-source-macos-integration:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "macOS"
+      artifact: "source"
+      flag: "TEST_INTEGRATION=1"
+
+  verify-rc-source-linux-java:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "ubuntu"
+      artifact: "source"
+      flag: "TEST_JAVA=1"
+
+  verify-rc-source-linux-csharp:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "ubuntu"
+      artifact: "source"
+      flag: "TEST_CSHARP=1"
+
+  verify-rc-source-linux-ruby:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "ubuntu"
+      artifact: "source"
+      flag: "TEST_RUBY=1"
+
+  verify-rc-source-linux-python:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "ubuntu"
+      artifact: "source"
+      flag: "TEST_PYTHON=1"
+
+  verify-rc-source-linux-js:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "ubuntu"
+      artifact: "source"
+      flag: "TEST_JS=1"
+
+  verify-rc-source-linux-go:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "ubuntu"
+      artifact: "source"
+      flag: "TEST_GO=1"
+
+  verify-rc-source-linux-rust:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "ubuntu"
+      artifact: "source"
+      flag: "TEST_RUST=1"
+
+  verify-rc-source-linux-integration:
+    ci: github
+    platform: linux
+    template: verify-rc/github.nix.yml
+    params:
+      os: "ubuntu"
+      artifact: "source"
+      flag: "TEST_INTEGRATION=1"
+
+  verify-rc-source-windows:
+    ci: github
+    platform: linux
+    template: verify-rc/github.windows.source.yml
+
+  verify-rc-wheels-windows:
+    ci: github
+    platform: linux
+    template: verify-rc/github.windows.wheels.yml
+
   ############################## Docker tests #################################
 
   test-conda-cpp: