Merge pull request #231 from natefoo/drop-py27-tests

Drop Python 2.7 support in standard transport, drop Python 2.7 tests and fix Python 3.7 wheel install test, general test debugging enhancements

.travis.yml

@@ -3,16 +3,10 @@ language: python
 dist: bionic
 matrix:
   include:
-  - python: 2.7
-    env: TOX_ENV=py27-lint
-  - python: 2.7
-    env: TOX_ENV=py27-lint-dist
-  - python: 2.7
+  - python: 3.7
+    env: TOX_ENV=py37-lint-dist
+  - python: 3.7
     env: TOX_ENV=lint-docs
-  - python: 2.7
-    env: TOX_ENV=py27 SETUP=true
-  - python: 2.7
-    env:  TOX_ENV=py27-install-wheel
   - python: 3.5
     env: TOX_ENV=py35-lint
   - python: 3.5
@@ -27,8 +21,8 @@ matrix:
     env: TOX_ENV=py37 SETUP=true
   - python: 3.7
     env: TOX_ENV=py37-unit SETUP=true
-  #- python: 3.7
-  #  env:  TOX_ENV=py37-install-wheel
+  - python: 3.7
+    env:  TOX_ENV=py37-install-wheel
 
 install:
   - pip install tox

Makefile

@@ -87,7 +87,10 @@ test-install-pypi:
 	bash install_test/test_install.bash
 
 test-install-wheel: dist
-	PULSAR_INSTALL_TARGET=$(shell pwd)/dist/pulsar_app*.whl bash install_test/test_install_conda.bash
+	PULSAR_INSTALL_TARGET=$(wildcard $(shell pwd)/dist/pulsar_app*.whl)[web] bash install_test/test_install_conda.bash
+
+test-install-wheel-no-conda: dist
+	PULSAR_INSTALL_TARGET=$(wildcard $(shell pwd)/dist/pulsar_app*.whl)[web] bash install_test/test_install.bash
 
 coverage:
 	coverage run --source $(SOURCE_DIR) setup.py $(TEST_DIR)

install_test/common_functions.bash

@@ -2,18 +2,25 @@ set -e
 
 shopt -s nullglob
 
-PULSAR_TARGET_POORT="${PULSAR_TARGET_POORT:-8913}"
-PULSAR_INSTALL_TARGET="${PULSAR_INSTALL_TARGET:-pulsar-app}"
-PLANEMO_INSTALL_TARGET="${PLANEMO_INSTALL_TARGET:-planemo==0.36.1}"
+: ${PULSAR_TARGET_PORT:=8913}
+: ${PULSAR_INSTALL_TARGET:=pulsar-app}
+: ${PULSAR_TEST_DEBUG:=false}
+: ${PLANEMO_INSTALL_TARGET:=planemo==0.72.0}
 
 init_temp_dir() {
-    TEMP_DIR=`mktemp -d`
+    case $(uname -s) in
+        Darwin)
+            TEMP_DIR=`mktemp -d -t pulsar-check-server`
+            ;;
+        *)
+            TEMP_DIR=`mktemp -d -t pulsar-check-server.XXXXXXXX`
+            ;;
+    esac
     echo "Setting up test directory $TEMP_DIR"
     cd "$TEMP_DIR"
 }
 
 init_pulsar() {
-    PULSAR_INSTALL_TARGET="${PULSAR_INSTALL_TARGET:-pulsar-app}"
     PROJECT_DIR="$SCRIPT_DIR/.."
 
     mkdir pulsar
@@ -22,7 +29,7 @@ init_pulsar() {
     virtualenv venv
     . venv/bin/activate # .venv\Scripts\activate if Windows
     echo "Installing Pulsar using 'pip install $PULSAR_INSTALL_TARGET'"
-    pip install $PULSAR_INSTALL_TARGET
+    pip install "$PULSAR_INSTALL_TARGET"
 
     cd ..
 }
@@ -39,23 +46,27 @@ stop_pulsar() {
 check_pulsar() {
     cd pulsar
 
-    if curl -s "http://localhost:$PULSAR_TARGET_POORT"
+    if curl -s "http://localhost:$PULSAR_TARGET_PORT"
     then
-        echo "Port $PULSAR_TARGET_POORT already bound, Pulsar will fail to start."
+        echo "Port $PULSAR_TARGET_PORT already bound, Pulsar will fail to start."
         exit 1;
     fi
 
     echo "Starting Pulsar in daemon mode."
     pulsar --daemon
     echo "Waiting for Pulsar to start."
-    while ! curl -s "http://localhost:$PULSAR_TARGET_POORT";
+    while ! curl -s "http://localhost:$PULSAR_TARGET_PORT";
     do
         printf "."
         sleep 1;
     done
     sleep 2
     echo "Running a standalone Pulsar job."
-    pulsar-check # runs a test job
+    if ! $PULSAR_TEST_DEBUG; then
+        pulsar-check # runs a test job
+    else
+        pulsar-check --debug --disable_cleanup
+    fi
     echo "Stopping Pulsar daemon."
     pulsar --stop-daemon
     echo "End Pulsar Checks"
@@ -64,7 +75,7 @@ check_pulsar() {
     echo "Starting Pulsar in daemon mode."
     pulsar --daemon
     echo "Waiting for Pulsar to start."
-    while ! curl -s "http://localhost:$PULSAR_TARGET_POORT";
+    while ! curl -s "http://localhost:$PULSAR_TARGET_PORT";
     do
         printf "."
         sleep 1;

install_test/test_install.bash

@@ -11,7 +11,7 @@ init_pulsar
 
 cd pulsar
 echo "Running pulsar-config with default arguments"
-pulsar-config --auto_conda
+pulsar-config
 cd ..
 
 check_pulsar

pulsar/client/test/check.py

@@ -4,6 +4,7 @@
 Exercises various features both the Pulsar client and server.
 """
 
+import logging
 import optparse
 import os
 import re
@@ -118,6 +119,7 @@ def assert_path_contents(path, expected_contents):
                         "is useful to checking the files generated during "
                         "the job and stored on the Pulsar server.")
 HELP_JOB_ID = "Submit the Pulsar job with this 'external' id."
+HELP_DEBUG = "Enable debug log output from Pulsar client"
 
 EXPECTED_OUTPUT = b"hello world output"
 EXAMPLE_UNICODE_TEXT = u'єχαмρℓє συтρυт'
@@ -136,9 +138,10 @@ def __init__(self, tool_dir):
 
 
 def run(options):
+    logging.basicConfig(level=logging.DEBUG)
     waiter = None
     try:
-        temp_directory = tempfile.mkdtemp()
+        temp_directory = tempfile.mkdtemp(prefix='pulsar-check-client.')
         temp_index_dir = os.path.join(temp_directory, "idx", "bwa")
         temp_index_dir_sibbling = os.path.join(temp_directory, "idx", "seq")
         temp_shared_dir = os.path.join(temp_directory, "shared", "test1")
@@ -284,7 +287,8 @@ def run(options):
     finally:
         if waiter is not None:
             waiter.shutdown()
-        shutil.rmtree(temp_directory)
+        if getattr(options, 'cleanup', True):
+            shutil.rmtree(temp_directory)
 
 
 class Waiter(object):
@@ -513,6 +517,7 @@ def main(argv=None):
     parser.add_option('--suppress_output', default=False, action="store_true", help=HELP_SUPPRESS_OUTPUT)
     parser.add_option('--disable_cleanup', dest="cleanup", default=True, action="store_false", help=HELP_DISABLE_CLEANUP)
     parser.add_option('--job_id', default="123456", help=HELP_JOB_ID)
+    parser.add_option('--debug', default=False, action="store_true", help=HELP_DEBUG)
     (options, args) = parser.parse_args(argv)
     run(options)
 

pulsar/client/transport/__init__.py

@@ -5,7 +5,7 @@
 from .requests import requests_multipart_post_available
 from .ssh import rsync_get_file, scp_get_file
 from .ssh import rsync_post_file, scp_post_file
-from .standard import Urllib2Transport
+from .standard import UrllibTransport
 
 if curl_available:
     from .curl import get_file
@@ -23,7 +23,7 @@ def get_transport(transport_type=None, os_module=os, transport_params=None):
     if not transport_params:
         transport_params = {}
     if transport_type == 'urllib':
-        transport = Urllib2Transport(**transport_params)
+        transport = UrllibTransport(**transport_params)
     else:
         transport = PycurlTransport(**transport_params)
     return transport

pulsar/client/transport/standard.py

@@ -1,5 +1,5 @@
 """
-Pulsar HTTP Client layer based on Python Standard Library (urllib2)
+Pulsar HTTP Client layer based on Python Standard Library (urllib)
 """
 
 from __future__ import with_statement
@@ -8,26 +8,20 @@
 import socket
 
 from os.path import getsize
-try:
-    from urllib2 import urlopen, URLError
-except ImportError:
-    from urllib.request import urlopen
-    from urllib.error import URLError
-try:
-    from urllib2 import Request
-except ImportError:
-    from urllib.request import Request
+from urllib.request import Request, urlopen
+from urllib.error import URLError
 
 from ..exceptions import PulsarClientTransportError
 
 
-class Urllib2Transport(object):
+class UrllibTransport(object):
 
     def __init__(self, timeout=None, **kwrgs):
         self.timeout = timeout
 
     def _url_open(self, request, data):
-        return urlopen(request, data, self.timeout)
+        # data is intentionally not used here (it is part of the request object), the parameter remains for tests
+        return urlopen(request, timeout=self.timeout)
 
     def execute(self, url, method=None, data=None, input_path=None, output_path=None):
         request = self.__request(url, data, method)
@@ -40,6 +34,9 @@ def execute(self, url, method=None, data=None, input_path=None, output_path=None
                     data = mmap.mmap(input.fileno(), 0, access=mmap.ACCESS_READ)
                 else:
                     data = b""
+                # setting the data property clears content-length, so the header must be set after (if content-length is
+                # unset, urllib sets transfer-encoding to chunked, which is not supported by webob on the server side).
+                request.data = data
                 request.add_header('Content-Length', str(size))
             try:
                 response = self._url_open(request, data)

pulsar/scripts/config.py

@@ -308,12 +308,11 @@ def _handle_app_yaml(args, directory):
         contents += 'private_token: %s\n' % args.private_token
     if args.mq:
         contents += 'message_queue_url: "amqp://guest:guest@localhost:5672//"\n'
-    if args.auto_conda:
-        contents += 'conda_auto_init: true\n'
-        contents += 'conda_auto_install: true\n'
-    else:
-        if not IS_WINDOWS and args.libdrmaa_path:
-            contents += 'manager:\n  type: queued_drmaa\n'
+    auto_conda = 'true' if args.auto_conda else 'false'
+    contents += 'conda_auto_init: {}\n'.format(auto_conda)
+    contents += 'conda_auto_install: {}\n'.format(auto_conda)
+    if not IS_WINDOWS and args.libdrmaa_path:
+        contents += 'manager:\n  type: queued_drmaa\n'
     open(yaml_file, "w").write(contents)
 
 

requirements.txt

@@ -8,10 +8,6 @@ galaxy-objectstore>=19.9.0.dev1
 galaxy-tool-util>=19.9.0.dev3
 paramiko
 
-# Problematic for Python 3.
-paste ; python_version == '2.7'
-PasteScript ; python_version == '2.7'
-
 ## Uncomment if using DRMAA queue manager.
 #drmaa
 

setup.py

@@ -115,6 +115,7 @@
     include_package_data=True,
     install_requires=requirements,
     extras_require={
+        'web': ['Paste', 'PasteScript'],
         ':python_version=="2.7"': py27_requirements,
         'galaxy_extended_metadata': ['galaxy-job-execution>=19.9.0.dev0', 'galaxy-util[template]'],
     },

test/client_test.py

@@ -6,7 +6,7 @@
 
 from pulsar.client.client import JobClient
 from pulsar.client.manager import HttpPulsarInterface
-from pulsar.client.transport import Urllib2Transport
+from pulsar.client.transport import UrllibTransport
 from pulsar.client.decorators import retry, MAX_RETRY_COUNT
 
 
@@ -43,7 +43,7 @@ def read(self, bytes=1024):
         return result
 
 
-class TestTransport(Urllib2Transport):
+class TestTransport(UrllibTransport):
     """ Implements mock of HTTP transport layer for TestClient tests."""
 
     def __init__(self, test_client):

test/client_transport_test.py

@@ -1,7 +1,7 @@
 import os
 from tempfile import NamedTemporaryFile
 
-from pulsar.client.transport.standard import Urllib2Transport
+from pulsar.client.transport.standard import UrllibTransport
 from pulsar.client.transport.curl import PycurlTransport
 from pulsar.client.transport.curl import post_file
 from pulsar.client.transport.curl import get_file
@@ -12,7 +12,7 @@
 
 
 def test_urllib_transports():
-    _test_transport(Urllib2Transport())
+    _test_transport(UrllibTransport())
 
 
 @skip_unless_module("pycurl")
@@ -96,8 +96,8 @@ def test_curl_problems():
 def test_get_transport():
     assert type(get_transport(None, FakeOsModule("1"))) == PycurlTransport
     assert type(get_transport(None, FakeOsModule("TRUE"))) == PycurlTransport
-    assert type(get_transport(None, FakeOsModule("0"))) == Urllib2Transport
-    assert type(get_transport('urllib', FakeOsModule("TRUE"))) == Urllib2Transport
+    assert type(get_transport(None, FakeOsModule("0"))) == UrllibTransport
+    assert type(get_transport('urllib', FakeOsModule("TRUE"))) == UrllibTransport
     assert type(get_transport('curl', FakeOsModule("TRUE"))) == PycurlTransport
 
 

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py27-lint, py27-lint-dist, lint-docs, py35-lint, py36-lint, py37-lint, py27, py27-unit, py35-unit, py36-unit, py37-unit
+envlist = py37-lint-dist, lint-docs, py35-lint, py36-lint, py37-lint, py35-unit, py36-unit, py37-unit
 toxworkdir={env:TOX_WORK_DIR:.tox}
 
 [testenv]
@@ -10,9 +10,6 @@ deps =
     drmaa
 passenv = DRMAA_LIBRARY_PATH
 
-[testenv:py27-unit]
-commands = nosetests --exclude '.*integration.*' []
-
 [testenv:py35]
 deps =
     -rrequirements.txt
@@ -40,11 +37,6 @@ deps =
     -rdev-requirements.txt
     drmaa
 
-[testenv:py27-lint]
-commands = flake8 --ignore W504 pulsar test
-skip_install = True
-deps = flake8
-
 [testenv:py35-lint]
 commands = flake8 --ignore W504 pulsar test
 skip_install = True
@@ -60,7 +52,7 @@ commands = flake8 --ignore W504 pulsar test
 skip_install = True
 deps = flake8
 
-[testenv:py27-lint-dist]
+[testenv:py37-lint-dist]
 commands = make lint-dist
 skip_install = True
 whitelist_externals = make
@@ -75,16 +67,16 @@ deps =
     -rrequirements.txt
     sphinx==1.2
 
-[testenv:py27-install-wheel]
+[testenv:py37-install-wheel]
 commands = make test-install-wheel
 skip_install = True
 whitelist_externals = make
 deps =
     virtualenv
 
-#[testenv:py37-install-wheel]
-#commands = make test-install-wheel
-#skip_install = True
-#whitelist_externals = make
-#deps =
-#    virtualenv
+[testenv:py37-install-wheel-no-conda]
+commands = make test-install-wheel-no-conda
+skip_install = True
+whitelist_externals = make
+deps =
+    virtualenv