ARROW-5511: [Packaging] Enable Flight in Conda packages

Author: Krisztián Szűcs <szucs.krisztian@gmail.com>
Author: Wes McKinney <wesm+git@apache.org>
Author: David Li <li.davidm96@gmail.com>

Closes #4473 from lihalite/conda-flight and squashes the following commits:

909b798 <Wes McKinney> Also build uriparser bundled in windows
55d2f41 <Wes McKinney> Build Flight in Windows conda packages
c1baa4c <Krisztián Szűcs> skip tests instead
c0b95ac <Krisztián Szűcs> debug env vars
e764a59 <Krisztián Szűcs> use SRC_DIR
f612508 <Krisztián Szűcs> export ARROW_TEST_DATA on the template level
8765af1 <Krisztián Szűcs> pass ARROW_TEST_DATA
d809fdf <Krisztián Szűcs> bundle uriparser
e9c4cd6 <David Li> Enable Flight in Conda packages

dev/tasks/conda-recipes/appveyor.win.yml

@@ -29,6 +29,7 @@ test_script:
   - git clone --no-checkout {{ arrow.remote }} arrow || exit /B
   - git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }} || exit /B
   - git -C arrow checkout FETCH_HEAD || exit /B
+  - git -C arrow submodule update --init --recursive || exit /B
 
   - pushd arrow\dev\tasks\conda-recipes
 

dev/tasks/conda-recipes/arrow-cpp/bld.bat

@@ -10,10 +10,12 @@ cmake -G "%CMAKE_GENERATOR%" ^
       -DARROW_BUILD_TESTS:BOOL=OFF ^
       -DARROW_BUILD_UTILITIES:BOOL=OFF ^
       -DCMAKE_BUILD_TYPE=release ^
+      -DARROW_FLIGHT:BOOL=ON ^
       -DARROW_PYTHON:BOOL=ON ^
       -DARROW_PARQUET:BOOL=ON ^
       -DARROW_GANDIVA:BOOL=ON ^
       -DARROW_ORC:BOOL=ON ^
+      -Duriparser_SOURCE=BUNDLED ^
       ..
 
 cmake --build . --target INSTALL --config Release

dev/tasks/conda-recipes/arrow-cpp/build.sh

@@ -19,12 +19,14 @@ cmake \
     -DCMAKE_INSTALL_LIBDIR=$PREFIX/lib \
     -DLLVM_TOOLS_BINARY_DIR=$PREFIX/bin \
     -DARROW_DEPENDENCY_SOURCE=SYSTEM \
+    -Duriparser_SOURCE=BUNDLED \
     -DARROW_PACKAGE_PREFIX=$PREFIX \
     -DARROW_BOOST_USE_SHARED=ON \
     -DARROW_BUILD_BENCHMARKS=OFF \
     -DARROW_BUILD_UTILITIES=OFF \
     -DARROW_BUILD_TESTS=OFF \
     -DARROW_JEMALLOC=ON \
+    -DARROW_FLIGHT=ON \
     -DARROW_PLASMA=ON \
     -DARROW_PYTHON=ON \
     -DARROW_PARQUET=ON \

dev/tasks/conda-recipes/arrow-cpp/meta.yaml

@@ -22,10 +22,12 @@ requirements:
   host:
     - boost-cpp
     - brotli
+    - c-ares
     - double-conversion
     - flatbuffers
     - gflags
     - glog
+    - grpc-cpp
     - libprotobuf
     - clangdev 7.*
     - llvmdev 7.*
@@ -44,9 +46,11 @@ requirements:
     - {{ pin_compatible('numpy', lower_bound='1.14') }}
     - boost-cpp
     - brotli
+    - c-ares
     - double-conversion
     - gflags
     - glog
+    - grpc-cpp
     - lz4-c
     - python
     - re2
@@ -58,6 +62,7 @@ test:
   commands:
     # headers
     - test -f $PREFIX/include/arrow/api.h              # [unix]
+    - test -f $PREFIX/include/arrow/flight/types.h     # [unix]
     - test -f $PREFIX/include/plasma/client.h          # [unix]
     - test -f $PREFIX/include/gandiva/engine.h         # [unix]
     - test -f $PREFIX/include/parquet/api/reader.h     # [unix]
@@ -67,17 +72,20 @@ test:
 
     # static
     - test -f $PREFIX/lib/libarrow.a         # [unix]
+    - test -f $PREFIX/lib/libarrow_flight.a  # [unix]
     - test -f $PREFIX/lib/libarrow_python.a  # [unix]
     - test -f $PREFIX/lib/libplasma.a        # [unix]
     - test -f $PREFIX/lib/libparquet.a       # [unix]
     - test -f $PREFIX/lib/libgandiva.a       # [unix]
     - if not exist %PREFIX%\\Library\\lib\\arrow_static.lib exit 1         # [win]
+    - if not exist %PREFIX%\\Library\\lib\\arrow_flight_static.lib exit 1  # [win]
     - if not exist %PREFIX%\\Library\\lib\\arrow_python_static.lib exit 1  # [win]
     - if not exist %PREFIX%\\Library\\lib\\parquet_static.lib exit 1       # [win]
     - if not exist %PREFIX%\\Library\\lib\\gandiva_static.lib exit 1       # [win]
 
     # shared
     - test -f $PREFIX/lib/libarrow.so            # [linux]
+    - test -f $PREFIX/lib/libarrow_flight.so     # [linux]
     - test -f $PREFIX/lib/libarrow_python.so     # [linux]
     - test -f $PREFIX/lib/libparquet.so          # [linux]
     - test -f $PREFIX/lib/libgandiva.so          # [linux]
@@ -88,6 +96,7 @@ test:
     - test -f $PREFIX/lib/libparquet.dylib       # [osx]
     - test -f $PREFIX/lib/libplasma.dylib        # [osx]
     - if not exist %PREFIX%\\Library\\bin\\arrow.dll exit 1         # [win]
+    - if not exist %PREFIX%\\Library\\bin\\arrow_flight.dll exit 1  # [win]
     - if not exist %PREFIX%\\Library\\bin\\arrow_python.dll exit 1  # [win]
     - if not exist %PREFIX%\\Library\\bin\\parquet.dll exit 1       # [win]
     - if not exist %PREFIX%\\Library\\bin\\gandiva.dll exit 1       # [win]

dev/tasks/conda-recipes/azure.linux.yml

@@ -23,6 +23,7 @@ jobs:
       git clone --no-checkout {{ arrow.remote }} arrow
       git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
       git -C arrow checkout FETCH_HEAD
+      git -C arrow submodule update --init --recursive
     displayName: Clone arrow
 
   - script: CI=azure ./run_docker_build.sh

dev/tasks/conda-recipes/azure.osx.yml

@@ -30,6 +30,7 @@ jobs:
       git clone --no-checkout {{ arrow.remote }} arrow
       git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
       git -C arrow checkout FETCH_HEAD
+      git -C arrow submodule update --init --recursive
     displayName: Clone arrow
 
   - script: |

dev/tasks/conda-recipes/azure.win.yml

@@ -52,6 +52,7 @@ jobs:
       git clone --no-checkout {{ arrow.remote }} arrow
       git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
       git -C arrow checkout FETCH_HEAD
+      git -C arrow submodule update --init --recursive
     displayName: Clone arrow
 
   # Configure the VM

dev/tasks/conda-recipes/pyarrow/bld.bat

@@ -14,10 +14,11 @@ copy /Y "%SRC_DIR%\cpp\cmake_modules\FindPythonLibsNew.cmake" cmake_modules\
 
 SET ARROW_HOME=%LIBRARY_PREFIX%
 SET SETUPTOOLS_SCM_PRETEND_VERSION=%PKG_VERSION%
+SET PYARROW_WITH_FLIGHT=1
+SET PYARROW_WITH_GANDIVA=1
+SET PYARROW_WITH_PARQUET=1
 "%PYTHON%" setup.py ^
            build_ext --build-type=release ^
-                     --with-parquet ^
-                     --with-gandiva ^
            install --single-version-externally-managed ^
                    --record=record.txt
 if errorlevel 1 exit 1

dev/tasks/conda-recipes/pyarrow/build.sh

@@ -12,6 +12,7 @@ cd python
 
 $PYTHON setup.py \
         build_ext --build-type=release \
+                  --with-flight \
                   --with-orc \
                   --with-plasma \
                   --with-parquet \

dev/tasks/conda-recipes/pyarrow/meta.yaml

@@ -45,6 +45,7 @@ requirements:
 test:
   imports:
     - pyarrow
+    - pyarrow.flight  # [py>=35]
     - pyarrow.plasma  # [unix]
     - pyarrow.parquet
     - pyarrow.gandiva  # [py>=35]
@@ -54,7 +55,7 @@ test:
     - hypothesis
 
   commands:
-    - pytest --pyargs pyarrow
+    - pytest -m "not requires_testing_data" --pyargs pyarrow
 
 about:
   home: http://github.com/apache/arrow

python/pyarrow/tests/test_flight.py

@@ -618,6 +618,7 @@ def test_location_invalid():
 
 
 @pytest.mark.slow
+@pytest.mark.requires_testing_data
 def test_tls_fails():
     """Make sure clients cannot connect when cert verification fails."""
     certs = example_tls_certs()
@@ -635,6 +636,7 @@ def test_tls_fails():
             client.do_get(flight.Ticket(b'ints'))
 
 
+@pytest.mark.requires_testing_data
 def test_tls_do_get():
     """Try a simple do_get call over TLS."""
     table = simple_ints_table()
@@ -650,6 +652,7 @@ def test_tls_do_get():
         assert data.equals(table)
 
 
+@pytest.mark.requires_testing_data
 def test_tls_override_hostname():
     """Check that incorrectly overriding the hostname fails."""
     certs = example_tls_certs()