THRIFT-3910 Do not invoke pip as part of build process

This closes apache#1073

appveyor.yml

@@ -71,6 +71,7 @@ build_script:
 # - set PATH=%PATH%;C:\Program Files (x86)\Haskell Platform\2014.2.0.0\bin
 # - set PATH=%PATH%;C:\Program Files (x86)\Haskell Platform\2014.2.0.0\lib\extralibs\bin
 - set PATH=C:\Python27-x64\scripts;C:\Python27-x64;%PATH%
+- pip install ipaddress backports.ssl_match_hostname tornado twisted
 - mkdir cmake-build
 - cd cmake-build
 - cmake -G "Visual Studio 14 2015 Win64" -DWITH_SHARED_LIB=OFF -DLIBEVENT_ROOT=C:\libevent-2.0.22-stable -DZLIB_INCLUDE_DIR=C:\zlib-1.2.8 -DZLIB_LIBRARY=C:\zlib-1.2.8\release\zlibstatic.lib -DBOOST_ROOT="%BOOST_ROOT%" -DBOOST_LIBRARYDIR="%BOOST_LIBRARYDIR%" ..

build/cmake/DefineOptions.cmake

@@ -103,9 +103,8 @@ endif()
 option(WITH_PYTHON "Build Python Thrift library" ON)
 find_package(PythonInterp QUIET) # for Python executable
 find_package(PythonLibs QUIET) # for Python.h
-find_package(Pip QUIET)
 CMAKE_DEPENDENT_OPTION(BUILD_PYTHON "Build Python library" ON
-                       "BUILD_LIBRARIES;WITH_PYTHON;PYTHONLIBS_FOUND;PIP_FOUND" OFF)
+                       "BUILD_LIBRARIES;WITH_PYTHON;PYTHONLIBS_FOUND" OFF)
 
 # Haskell
 option(WITH_HASKELL "Build Haskell Thrift library" ON)

build/docker/centos/Dockerfile

@@ -56,9 +56,11 @@ RUN yum install -y \
 # Python Dependencies
 RUN yum install -y \
       python-devel \
+      python-pip \
       python-setuptools \
-      python-twisted-web \
-      python-pip
+      python-six \
+      python-twisted-web && \
+    pip install -U backports.ssl_match_hostname ipaddress tornado
 
 # Ruby Dependencies
 RUN yum install -y \

build/docker/centos6/Dockerfile

@@ -40,6 +40,9 @@ RUN yum install -y epel-release && \
       python-pip \
     && yum clean all
 
+# optional dependencies
+RUN pip install ipaddress backports.ssl_match_hostname tornado
+
 # CMake
 RUN curl -sSL https://cmake.org/files/v3.4/cmake-3.4.1.tar.gz | tar -xz && \
     cd cmake-3.4.1 && ./bootstrap && make -j4 && make install && \

build/docker/debian/Dockerfile

@@ -145,6 +145,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # Ruby
 RUN gem install bundler --no-ri --no-rdoc
 
+# Python optional dependencies
+RUN pip2 install -U ipaddress backports.ssl_match_hostname tornado
+RUN pip3 install -U backports.ssl_match_hostname tornado
+
 # Go
 RUN curl -sSL https://storage.googleapis.com/golang/go1.4.3.linux-amd64.tar.gz | tar -C /usr/local/ -xz
 ENV PATH /usr/local/go/bin:$PATH

build/docker/ubuntu/Dockerfile

@@ -163,6 +163,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 # Ruby
 RUN gem install bundler --no-ri --no-rdoc
 
+# Python optional dependencies
+RUN pip2 install -U ipaddress backports.ssl_match_hostname tornado
+RUN pip3 install -U backports.ssl_match_hostname tornado
+
 # Go
 RUN curl -sSL https://storage.googleapis.com/golang/go1.4.3.linux-amd64.tar.gz | tar -C /usr/local/ -xz
 ENV PATH /usr/local/go/bin:$PATH

configure.ac

@@ -280,20 +280,25 @@ AM_CONDITIONAL(WITH_LUA, [test "$have_lua" = "yes"])
 AM_PATH_PYTHON(2.6,, :)
 AX_THRIFT_LIB(python, [Python], yes)
 if test "$with_python" = "yes";  then
-  AC_PATH_PROG([PIP], [pip])
-  AC_PATH_PROG([TRIAL], [trial])
-  if test -n "$TRIAL" && test "x$PYTHON" != "x" && test "x$PYTHON" != "x:" ; then
+  if test -n "$PYTHON"; then
     have_python="yes"
   fi
+  AC_PATH_PROG([TRIAL], [trial])
+  if test -n "$TRIAL"; then
+    have_trial="yes"
+  fi
 fi
 AM_CONDITIONAL(WITH_PYTHON, [test "$have_python" = "yes"])
+AM_CONDITIONAL(WITH_TWISTED_TEST, [test "$have_trial" = "yes"])
 
 # Find "python3" executable.
 # It's distro specific and far from ideal but needed to cross test py2-3 at once.
-AC_PATH_PROG([PYTHON3], [python3])
-AC_PATH_PROG([PIP3], [pip3])
-if test "x$PYTHON3" != "x" && test "x$PYTHON3" != "x:" && test "x$PIP3" != "x" ; then
-  have_py3="yes"
+# TODO: find "python2" if it's 3.x
+if python --version 2>&1 | grep -q "Python 2"; then
+  AC_PATH_PROGS([PYTHON3], [python3 python3.5 python35 python3.4 python34])
+  if test -n "$PYTHON3"; then
+    have_py3="yes"
+  fi
 fi
 AM_CONDITIONAL(WITH_PY3, [test "$have_py3" = "yes"])
 
@@ -859,7 +864,12 @@ if test "$have_python" = "yes" ; then
   echo
   echo "Python Library:"
   echo "   Using Python .............. : $PYTHON"
-  echo "   Using Trial ............... : $TRIAL"
+  if test "$have_py3" = "yes" ; then
+  echo "   Using Python3 ............. : $PYTHON3"
+  fi
+  if test "$have_trial" = "yes"; then
+  echo "   Using trial ............... : $TRIAL"
+  fi
 fi
 if test "$have_php" = "yes" ; then
   echo

lib/py/CMakeLists.txt

@@ -20,7 +20,6 @@
 include_directories(${PYTHON_INCLUDE_DIRS})
 
 add_custom_target(python_build ALL
-    COMMAND ${PIP_EXECUTABLE} install -r requirements.txt || ${PIP_EXECUTABLE} install --user -r requirements.txt
     COMMAND ${PYTHON_EXECUTABLE} setup.py build
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
     COMMENT "Building Python library"

lib/py/Makefile.am

@@ -21,7 +21,6 @@ DESTDIR ?= /
 
 if WITH_PY3
 py3-build:
-	$(PIP3) install -r requirements.txt || $(PIP3) install --user -r requirements.txt
 	$(PYTHON3) setup.py build
 py3-test: py3-build
 	$(PYTHON3) test/thrift_json.py
@@ -32,15 +31,13 @@ py3-test:
 endif
 
 all-local: py3-build
-	$(PIP) install -r requirements.txt || $(PIP) install --user -r requirements.txt
 	$(PYTHON) setup.py build
 
 # We're ignoring prefix here because site-packages seems to be
 # the equivalent of /usr/local/lib in Python land.
 # Old version (can't put inline because it's not portable).
 #$(PYTHON) setup.py install --prefix=$(prefix) --root=$(DESTDIR) $(PYTHON_SETUPUTIL_ARGS)
 install-exec-hook:
-	$(PIP) install -r requirements.txt
 	$(PYTHON) setup.py install --root=$(DESTDIR) --prefix=$(PY_PREFIX) $(PYTHON_SETUPUTIL_ARGS)
 
 clean-local:
@@ -54,7 +51,6 @@ EXTRA_DIST = \
 	CMakeLists.txt \
 	coding_standards.md \
 	compat \
-	requirements.txt \
 	setup.py \
 	setup.cfg \
 	src \

lib/py/setup.py

@@ -78,6 +78,14 @@ def run_setup(with_binary):
     else:
         extensions = dict()
 
+    ssl_deps = []
+    if sys.version_info[0] == 2:
+        ssl_deps.append('ipaddress')
+    if sys.hexversion < 0x03050000:
+        ssl_deps.append('backports.ssl_match_hostname>=3.5')
+    tornado_deps = ['tornado>=4.0']
+    twisted_deps = ['twisted']
+
     setup(name='thrift',
           version='1.0.0-dev',
           description='Python bindings for the Apache Thrift RPC system',
@@ -86,6 +94,12 @@ def run_setup(with_binary):
           url='http://thrift.apache.org',
           license='Apache License 2.0',
           install_requires=['six>=1.7.2'],
+          extras_require={
+              'ssl': ssl_deps,
+              'tornado': tornado_deps,
+              'twisted': twisted_deps,
+              'all': ssl_deps + tornado_deps + twisted_deps,
+          },
           packages=[
               'thrift',
               'thrift.protocol',

lib/py/src/transport/TSSLSocket.py

@@ -346,7 +346,7 @@ def __init__(self, host=None, port=9090, *args, **kwargs):
         TSSLBase.__init__(self, True, None, kwargs)
         TSocket.TServerSocket.__init__(self, host, port, unix_socket)
         if self._should_verify and not _match_has_ipaddress:
-            raise ValueError('Need ipaddress and backports.ssl_match_hostname'
+            raise ValueError('Need ipaddress and backports.ssl_match_hostname '
                              'module to verify client certificate')
 
     def setCertfile(self, certfile):

test/Makefile.am

@@ -53,8 +53,10 @@ endif
 if WITH_PYTHON
 SUBDIRS += py
 PRECROSS_TARGET += precross-py
-SUBDIRS += py.twisted
 SUBDIRS += py.tornado
+if WITH_TWISTED_TEST
+SUBDIRS += py.twisted
+endif
 endif
 
 if WITH_RUBY

test/py.twisted/Makefile.am

@@ -18,6 +18,7 @@
 #
 
 THRIFT = $(top_builddir)/compiler/cpp/thrift
+TRIAL ?= trial
 
 stubs: ../ThriftTest.thrift ../SmallTest.thrift
 	$(THRIFT) --gen py:twisted ../ThriftTest.thrift