Libssh2 upgrade (#101)

Updated embedded libbsh2 version to 1.9.0
Re-generated C files with latest cython
Updated appveyor and travis cfgs
Updated docker wheel build for new libssh2 version
Updated gitignore
Updated changelog, travis cfg, readme
Removed windows python 3.5 and 32 bit builds

.appveyor.yml

@@ -12,57 +12,22 @@ environment:
     PYPI_PASS:
       secure: x+dF0A8BZUf2IrPNRN1O0w==
   matrix:
-    - PYTHON: "C:\\Python27"
-      PYTHON_VERSION: "2.7"
-      PYTHON_ARCH: "32"
-      MSVC: "Visual Studio 9"
-      ARCH: i386
-      SYSTEM_LIBSSH2: 1
-
-    - PYTHON: "C:\\Python27-x64"
-      PYTHON_VERSION: "2.7"
-      PYTHON_ARCH: "64"
-      MSVC: "Visual Studio 9"
-      ARCH: x64_86
-      SYSTEM_LIBSSH2: 1
-
-    - PYTHON: "C:\\Python35"
-      PYTHON_VERSION: "3.5"
-      PYTHON_ARCH: "32"
-      MSVC: "Visual Studio 14"
-      ARCH: i386
-      SYSTEM_LIBSSH2: 1
-
-    - PYTHON: "C:\\Python35-x64"
-      PYTHON_VERSION: "3.5"
-      PYTHON_ARCH: "64"
-      MSVC: "Visual Studio 14 Win64"
-      ARCH: x64_86
-      SYSTEM_LIBSSH2: 1
-
-    - PYTHON: "C:\\Python36"
-      PYTHON_VERSION: "3.6"
-      PYTHON_ARCH: "32"
-      MSVC: "Visual Studio 14"
-      ARCH: i386
-      SYSTEM_LIBSSH2: 1
-
     - PYTHON: "C:\\Python36-x64"
       PYTHON_VERSION: "3.6"
       PYTHON_ARCH: "64"
       MSVC: "Visual Studio 14 Win64"
       ARCH: x64_86
       SYSTEM_LIBSSH2: 1
 
-    - PYTHON: "C:\\Python37"
+    - PYTHON: "C:\\Python37-x64"
       PYTHON_VERSION: "3.7"
-      PYTHON_ARCH: "32"
-      MSVC: "Visual Studio 14"
-      ARCH: i386
+      PYTHON_ARCH: "64"
+      MSVC: "Visual Studio 14 Win64"
+      ARCH: x64_86
       SYSTEM_LIBSSH2: 1
 
-    - PYTHON: "C:\\Python37-x64"
-      PYTHON_VERSION: "3.7"
+    - PYTHON: "C:\\Python38-x64"
+      PYTHON_VERSION: "3.8"
       PYTHON_ARCH: "64"
       MSVC: "Visual Studio 14 Win64"
       ARCH: x64_86

.gitignore

@@ -4,5 +4,7 @@ dist
 build
 *~
 *.so
+src
+wheelhouse
 .idea/
 ssh2/libssh2.so*

.travis.yml

@@ -12,9 +12,9 @@ services:
   - docker
 python:
   - 2.7
-  - 3.5
   - 3.6
   - 3.7
+  - 3.8
 addons:
   apt:
     packages:
@@ -45,38 +45,24 @@ jobs:
     - &osx-wheels
       stage: build packages
       os: osx
-      osx_image: xcode9.2
+      osx_image: xcode11.6
       env:
+        - PYENV: 3.6.11
         - SYSTEM_LIBSSH2: 1
       before_cache:
         - brew cleanup
       before_install:
         - brew update
         - brew install ccache
         - brew outdated openssl || travis_wait brew upgrade openssl || echo "y"
-        - brew link --overwrite python@2 || brew install python@2 || brew link --overwrite python@2
-        - which python2
-        - python2 -c "from __future__ import print_function; import ssl; from platform import python_version; print(ssl.OPENSSL_VERSION); print(python_version())"
-        - sudo -H pip2 install twine
+        - sudo -H pip install twine
         - which twine
         - sudo ci/install-ssh2.sh
         - cp /usr/local/lib/libssh2* .
         - mkdir -p wheels
-      install:
-        - sudo -H pip2 install -U delocate wheel pip setuptools
-        - ls -lh
+      install: skip
       script:
-        - pip2 wheel .
-        - delocate-listdeps --all *.whl
-        - delocate-wheel -v *.whl
-        - delocate-listdeps --all *.whl
-        - ls -l *.whl
-        - rm -f *.dylib
-        - pip2 install --user -v *.whl
-        - pwd; mkdir temp; cd temp; pwd
-        - python -c "from ssh2.session import Session; Session()"
-        - cd ..; pwd
-        - mv -f *.whl wheels/
+        - travis_wait ./ci/travis/pyenv-wheel.sh
       after_success:
         - if [[ ! -z "$TRAVIS_TAG" ]]; then
               twine upload --skip-existing -u $PYPI_U -p $PYPI_P wheels/*.whl;
@@ -85,39 +71,45 @@ jobs:
       python: skip
 
     - <<: *osx-wheels
-      osx_image: xcode9.2
+      osx_image: xcode11.3
+      env:
+        - PYENV: 3.6.11
+        - SYSTEM_LIBSSH2: 1
+      install: skip
+      script:
+        - travis_wait ./ci/travis/pyenv-wheel.sh
 
     - <<: *osx-wheels
-      osx_image: xcode9.2
+      osx_image: xcode11.6
       env:
-        - PYENV: 3.6.4
+        - PYENV: 3.7.8
         - SYSTEM_LIBSSH2: 1
       install: skip
       script:
         - travis_wait ./ci/travis/pyenv-wheel.sh
 
     - <<: *osx-wheels
-      osx_image: xcode9.4
+      osx_image: xcode11.3
       env:
-        - PYENV: 3.6.4
+        - PYENV: 3.7.8
         - SYSTEM_LIBSSH2: 1
       install: skip
       script:
         - travis_wait ./ci/travis/pyenv-wheel.sh
 
     - <<: *osx-wheels
-      osx_image: xcode9.2
+      osx_image: xcode11.3
       env:
-        - PYENV: 3.7.0
+        - PYENV: 3.8.5
         - SYSTEM_LIBSSH2: 1
       install: skip
       script:
         - travis_wait ./ci/travis/pyenv-wheel.sh
 
     - <<: *osx-wheels
-      osx_image: xcode9.4
+      osx_image: xcode11.6
       env:
-        - PYENV: 3.7.0
+        - PYENV: 3.8.5
         - SYSTEM_LIBSSH2: 1
       install: skip
       script:

Changelog.rst

@@ -1,6 +1,27 @@
 Change Log
 =============
 
+0.19.0
++++++++
+
+Changes
+--------
+
+* Updated embedded libssh2 version to ``1.9.0``.
+* Rebuilt sources with Cython ``0.29.21``.
+* Added support for Python 3.8 and 3.9.
+
+Packaging
+----------
+
+* Added Python 3.8 binary wheels for Linux, OSX and Windows.
+* Added Python 3.9 binary wheels for Linux.
+* Added OSX 10.14 and 10.15 wheels.
+* Removed OSX < 10.14 wheels.
+* Removed Python 2.7 OSX and Windows wheels.
+* Removed Python 3.5 OSX and Windows wheels.
+* Removed Windows 32-bit wheels.
+
 0.18.0
 +++++++
 

README.rst

@@ -33,8 +33,6 @@ Binary wheel packages are provided for Linux, OSX and Windows, all Python versio
 
    pip install ssh2-python
 
-`Conda <https://conda.io/miniconda.html>`_ is another installation option - see `documentation <http://ssh2-python.readthedocs.org/en/latest/>`_ for more detailed instructions.
-
 For from source installation instructions, including building against system provided libssh2, `see documentation <https://ssh2-python.readthedocs.io/en/latest/installation.html#installation-from-source>`_.
 
 For creating native system packages for Centos/RedHat, Ubuntu, Debian and Fedora, see `instructions in the documentation <http://ssh2-python.readthedocs.io/en/latest/installation.html#system-binary-packages>`_.
@@ -57,7 +55,7 @@ The library uses `Cython`_ based native code extensions as wrappers to ``libssh2
 
 Extension features:
 
-* Thread safe - GIL is released as much as possible
+* Thread safe - GIL is released as much as possible. Note that libssh2 does not support sharing sessions across threads
 * Very low overhead
 * Super fast as a consequence of the excellent C library it uses and prodigious use of native code
 * Object oriented - memory freed automatically and safely as objects are garbage collected by Python

ci/docker/manylinux/Dockerfile

@@ -3,10 +3,11 @@ FROM quay.io/pypa/manylinux1_x86_64
 ENV CMAKE cmake-2.8.11.1-5.4.x86_64
 ENV OPENSSL openssl-1.1.1b
 ENV SYSTEM_LIBSSH2 1
+ENV LIBSSH2_VERSION 1.9.0
 
 RUN yum install zlib-devel -y
 
-ADD libssh2.tar.gz libssh2.tar.gz
+ADD libssh2-${LIBSSH2_VERSION}.tar.gz libssh2-${LIBSSH2_VERSION}.tar.gz
 ADD ${CMAKE}.rpm cmake.rpm
 ADD ${OPENSSL}.tar.gz ${OPENSSL}.tar.gz
 ADD local-perl-5.10.0-62.ep.x86_64.rpm local-perl.rpm
@@ -26,7 +27,8 @@ RUN cd ${OPENSSL}.tar.gz/${OPENSSL} && \
 
 # Libssh2
 RUN mkdir -p build_libssh2 && cd build_libssh2 && \
-    cmake ../libssh2.tar.gz/libssh2-master -DBUILD_SHARED_LIBS=ON -DENABLE_ZLIB_COMPRESSION=ON \
+    cmake ../libssh2-${LIBSSH2_VERSION}.tar.gz/libssh2-libssh2-${LIBSSH2_VERSION} \
+    -DBUILD_SHARED_LIBS=ON -DENABLE_ZLIB_COMPRESSION=ON \
     -DENABLE_CRYPT_NONE=ON -DENABLE_MAC_NONE=ON -DCMAKE_INSTALL_PREFIX=/usr && \
     cmake --build . --config Release --target install
 

ci/travis/build-manylinux.sh

@@ -3,9 +3,10 @@
 docker_tag="parallelssh/ssh2-manylinux"
 
 rm -rf build dist ssh2/libssh2.*
+python ci/appveyor/fix_version.py .
 
 docker pull $docker_tag || echo
 docker build --cache-from $docker_tag ci/docker/manylinux -t $docker_tag
 if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then docker push $docker_tag; fi
-docker run --rm -v `pwd`:/io $docker_tag /io/ci/travis/build-wheels.sh
+docker run -e TRAVIS_TAG="$TRAVIS_TAG" --rm -v `pwd`:/io $docker_tag /io/ci/travis/build-wheels.sh
 ls wheelhouse/

ci/travis/build-wheels.sh

@@ -1,6 +1,10 @@
 #!/bin/bash -xe
 
+echo "Travis tag: $TRAVIS_TAG"
+
 # Compile wheels
+# For testing
+# for PYBIN in `ls -1d /opt/python/cp27-cp27m/bin | grep -v cpython`; do
 for PYBIN in `ls -1d /opt/python/*/bin | grep -v cpython`; do
     "${PYBIN}/pip" wheel /io/ -w wheelhouse/
 done
@@ -11,7 +15,9 @@ for whl in wheelhouse/*.whl; do
 done
 
 # Install packages and test
-for PYBIN in `ls -1d /opt/python/*/bin | grep -v cpython`; do
-    "${PYBIN}/pip" install ssh2-python --no-index -f /io/wheelhouse
-    (cd "$HOME"; "${PYBIN}/python" -c 'from ssh2.session import Session; Session()')
-done
+if [[ ! -z "$TRAVIS_TAG" ]]; then
+    for PYBIN in `ls -1d /opt/python/*/bin | grep -v cpython`; do
+        "${PYBIN}/pip" install ssh2-python --no-index -f /io/wheelhouse
+        (cd "$HOME"; "${PYBIN}/python" -c 'from ssh2.session import Session; Session()')
+    done
+fi

libssh2/Makefile.am

@@ -147,3 +147,6 @@ $(VCPROJ): win32/vc8proj.head win32/vc8proj.foot Makefile.am
 	done; \
 	cat $(srcdir)/vc8proj.foot) | \
 	awk '{printf("%s\r\n", gensub("\r", "", "g"))}' > $@ )
+
+checksrc:
+	perl src/checksrc.pl -i4 -m79 -ASIZEOFNOPAREN -ASNPRINTF -ACOPYRIGHT -AFOPENMODE -Wsrc/libssh2_config.h src/*.[ch] include/*.h example/*.c

libssh2/RELEASE-NOTES

@@ -1,31 +1,44 @@
-libssh2 1.8.0
+libssh2 1.9.0
 
-This release includes the following changes:
-
- o added a basic dockerised test suite
- o crypto: add support for the mbedTLS backend
-
-This release includes the following bugfixes:
-
- o libgcrypt: fixed a NULL pointer dereference on OOM
- o VMS: can't use %zd for off_t format
- o VMS: update vms/libssh2_config.h
- o windows: link with crypt32.lib
- o libssh2_channel_open: speeling error fixed in channel error message
- o msvc: fixed 14 compilation warnings
- o tests: HAVE_NETINET_IN_H was not defined correctly
- o openssl: add OpenSSL 1.1.0 compatibility
- o cmake: Add CLEAR_MEMORY option, analogously to that for autoconf
- o configure: make the --with-* options override the OpenSSL default
- o libssh2_wait_socket: set err_msg on errors
- o libssh2_wait_socket: Fix comparison with api_timeout to use milliseconds
+This release includes the following enhancements and bugfixes:
+
+ o adds ECDSA keys and host key support when using OpenSSL
+ o adds ED25519 key and host key support when using OpenSSL 1.1.1
+ o adds OpenSSH style key file reading
+ o adds AES CTR mode support when using WinCNG
+ o adds PEM passphrase protected file support for Libgcrypt and WinCNG
+ o adds SHA256 hostkey fingerprint
+ o adds libssh2_agent_get_identity_path() and libssh2_agent_set_identity_path()
+ o adds explicit zeroing of sensitive data in memory
+ o adds additional bounds checks to network buffer reads
+ o adds the ability to use the server default permissions when creating sftp directories
+ o adds support for building with OpenSSL no engine flag
+ o adds support for building with LibreSSL
+ o increased sftp packet size to 256k
+ o fixed oversized packet handling in sftp
+ o fixed building with OpenSSL 1.1
+ o fixed a possible crash if sftp stat gets an unexpected response
+ o fixed incorrect parsing of the KEX preference string value
+ o fixed conditional RSA and AES-CTR support
+ o fixed a small memory leak during the key exchange process
+ o fixed a possible memory leak of the ssh banner string
+ o fixed various small memory leaks in the backends
+ o fixed possible out of bounds read when parsing public keys from the server
+ o fixed possible out of bounds read when parsing invalid PEM files
+ o no longer null terminates the scp remote exec command
+ o now handle errors when diffie hellman key pair generation fails
+ o fixed compiling on Windows with the flag STDCALL=ON
+ o improved building instructions
+ o improved unit tests
 
 This release would not have looked like this without help, code, reports and
 advice from friends like these:
 
-  Alexander Lamaison, Antenore Gatta, Brad Harder, Charles Collicutt,
-  Craig A. Berry, Dan Fandrich, Daniel Stenberg, Kamil Dudka, Keno Fischer,
-  Taylor Holberton, Viktor Szakats, Will Cosgrove, Zenju
-  (12 contributors)
-
-        Thanks! (and sorry if I forgot to mention someone)
+  Peter Surge, Will Cosgrove, Daniel Stenberg, Alex Arslan, Alex Crichton, 
+  Thomas Bleeker, Keno Fischer, Marc Hörsken, Marcel Raad, Viktor Szakats,
+  Kamil Dudka, Panos, Etienne Samson, Tseng Jun, Brendan Shanks, doublex,
+  Erik B, Jakob Egger, Thomas Lochmatter, alex-weaver, Adrian Moran, Zenju,
+  gartens, Matthew D. Fuller, Ryan Kelley, Zhen-Huan HWANG, Orivej Desh, 
+  Alexander Curtiss
+
+  (29 contributors)