Try conda/poetry for dev-env

.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+repos:
+    - repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v3.1.0
+      hooks:
+        - id: check-yaml
+          args: [--unsafe]
+          exclude: '.*templates.*.yaml'
+        - id: end-of-file-fixer
+          exclude: 'CHANGELOG.md'
+        - id: trailing-whitespace
+          exclude: 'CHANGELOG.md'
+        - id: check-ast
+    - repo: https://github.com/gruntwork-io/pre-commit
+      rev: v0.1.9
+      hooks:
+        - id: shellcheck
+    - repo: https://github.com/psf/black
+      rev: 19.10b0
+      hooks:
+        - id: black

.travis.yml

@@ -10,35 +10,22 @@ python:
 # moto is incompatible with 3.5 because it relies on the key order of its url path regex rules
 #    - 3.5.9
 
-matrix:
-  include:
-  - python: 3.7
-    env: EXTRA="awscli"
-  - python: 3.7
-    env: EXTRA="boto3"
-
 install:
-    # unfortunately pipenv check fails so often due to pyup.io/security key we need to ignore it :(
-    - pip install -U setuptools pip
-    - if ! [[ -v EXTRA ]]; then
-        pip install -U pipenv &&
-        pipenv lock &&
-        pipenv sync --dev &&
-        (pipenv check || true) &&
-        pipenv graph;
-      else
-        pip install codecov &&
-        pip check &&
-        pip freeze;
-      fi
+    ## Install project dependencies into cached venv
+    ## https://docs.gitlab.com/ee/ci/caching/#caching-python-dependencies
+    - python -V
+    - pip install -U pip setuptools virtualenv
+    - virtualenv venv
+    - source venv/bin/activate
+    - curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python
+    - source $HOME/.poetry/env
+    - poetry run python -m pip install -r requirements.dev
+    - poetry install -v --no-interaction --extras all
+    - poetry show -t --no-dev
 
 script:
-    - if [[ -v EXTRA ]]; then
-        pip install -U pipdeptree .[$EXTRA] &&
-        pipdeptree;
-      else
-        make flake mototest;
-      fi
+    - source $HOME/.poetry/env
+    - make mototest
 after_success:
     codecov
 deploy:

CONTRIBUTING.rst

@@ -17,12 +17,14 @@ Create virtualenv with at least python3.5 (older version are not supported).
 For example using *virtualenvwrapper* commands could look like::
 
    $ cd aiobotocore
-   $ mkvirtualenv --python=`which python3.5` aiobotocore
-
-
-After that please install libraries required for development::
-
-    $ pipenv sync --dev
+   $ make init
+
+The virtualenv is provided by a conda installation and the project
+dependencies are installed by poetry.  See the `Makefile` and the
+`init.sh` script for details.  The `make init` script is designed
+to be run only when a clean virtualenv is created; after that, the
+virtualenv is usually only activated and packages are managed using
+poetry.
 
 Congratulations, you are ready to run the test suite::
 

Makefile

@@ -2,37 +2,40 @@
 
 FLAGS=
 
-flake: checkrst
-	pipenv run python3 -m flake8 --format=abspath
+LIB=aiobotocore
+
+# the install uses conda to provide a python 3.6 base environment;
+# the init.sh script uses a reliable conda environment to do so.
+init: poetry
+	./init.sh
+
+flake: package-check
+	@poetry run python3 -m flake8 --format=abspath
 
 test: flake
-	pipenv run python3 -Wd -m pytest -s -vv $(FLAGS) ./tests/
+	@poetry run python3 -Wd -m pytest -s -vv $(FLAGS) ./tests/
 
 vtest:
-	pipenv run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -s -vv $(FLAGS) ./tests/
-
-checkrst:
-	pipenv run python3 setup.py check -rms
+	@poetry run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -s -vv $(FLAGS) ./tests/
 
 cov cover coverage: flake
-	pipenv run python3 -Wd -m pytest -s -vv --cov-report term --cov-report html --cov aiobotocore ./tests
+	@poetry run python3 -Wd -m pytest -s -vv --cov-report term --cov-report html --cov $(LIB) ./tests
 	@echo "open file://`pwd`/htmlcov/index.html"
 
 # BOTO_CONFIG solves https://github.com/travis-ci/travis-ci/issues/7940
 mototest:
-	BOTO_CONFIG=/dev/null pipenv run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -vv -m moto -n auto --cov-report term --cov-report html --cov aiobotocore tests
+	BOTO_CONFIG=/dev/null poetry run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -vv -m moto -n auto --cov-report term --cov-report html --cov $(LIB) tests
 	@echo "open file://`pwd`/htmlcov/index.html"
 
-
 clean:
-	rm -rf `find . -name __pycache__`
-	rm -f `find . -type f -name '*.py[co]' `
-	rm -f `find . -type f -name '*~' `
-	rm -f `find . -type f -name '.*~' `
-	rm -f `find . -type f -name '@*' `
-	rm -f `find . -type f -name '#*#' `
-	rm -f `find . -type f -name '*.orig' `
-	rm -f `find . -type f -name '*.rej' `
+	find . -name __pycache__ -exec rm -rf {} +
+	find . -type f -name '*.py[co]' -exec rm -f {} +
+	find . -type f -name '*~' -exec rm -f {} +
+	find . -type f -name '.*~' -exec rm -f {} +
+	find . -type f -name '@*' -exec rm -f {} +
+	find . -type f -name '#*#' -exec rm -f {} +
+	find . -type f -name '*.orig' -exec rm -f {} +
+	find . -type f -name '*.rej' -exec rm -f {} +
 	rm -f .coverage
 	rm -rf coverage
 	rm -rf build
@@ -43,4 +46,26 @@ doc:
 	make -C docs html
 	@echo "open file://`pwd`/docs/_build/html/index.html"
 
+typehint: clean
+	@poetry run mypy --follow-imports=skip $(LIB) tests
+
+package: clean
+	@poetry check
+	@poetry build
+
+package-check: package
+	@poetry run twine check dist/*
+
+poetry:
+	@if ! command -v poetry > /dev/null; then \
+		curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py -o /tmp/get-poetry.py; \
+		python /tmp/get-poetry.py; \
+	fi
+
+poetry-export:
+	poetry export --without-hashes -f requirements.txt -o requirements.txt
+	sed -i -e 's/^-e //g' requirements.txt
+
+
+.PHONY: init typehint package package-check poetry poetry-export
 .PHONY: all flake test vtest cov clean doc

conda_venv.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+# Copyright 2019-2020 Darren Weber
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# https://python-release-cycle.glitch.me/
+
+# Source this file from ~/.bashrc or similar shell-init, such
+# as copy the file to /etc/profile.d/
+
+
+# Enable this to create conda env for each python version
+if false; then
+    conda create -y -n py3.6 python=3.6
+    conda create -y -n py3.7 python=3.7
+    conda create -y -n py3.8 python=3.8
+fi
+
+# CONDA PYTHON VERSIONS
+alias py36='conda deactivate; conda activate py3.6'
+alias py37='conda deactivate; conda activate py3.7'
+alias py38='conda deactivate; conda activate py3.8'
+
+conda-project () {
+    # The project name is defined by CONDA_ENV or the current working directory
+    project=${CONDA_ENV:-$(pwd)}
+    basename "${project}"
+}
+
+conda-venv-activate () {
+    # try to activate a conda environment with the name of
+    # the current directory (often this is a project name).
+    wd=$(conda-project)
+    conda deactivate
+    conda activate "$wd"
+}
+
+conda-venv-create () {
+    # create and activate a conda environment with the name
+    # of the current directory (often this is a project name).
+    py_ver="${1:-3.6}"
+    wd=$(conda-project)
+    conda deactivate
+
+    conda create -n "$wd" python="${py_ver}" \
+      --channel conda-forge --override-channels \
+    && conda activate "$wd" \
+    && conda config --env --add channels conda-forge \
+    && conda config --env --set channel_priority strict
+}
+
+conda-venv-remove () {
+    # try to activate a conda environment with the name of
+    # the current directory (often this is a project name).
+    wd=$(conda-project)
+    conda deactivate
+    conda env remove -n "$wd"
+}
+
+conda-venv () {
+    # create and activate a conda environment with the name
+    # of the current directory (often this is a project name).
+    py_ver="${1:-3.6}"
+    wd=$(conda-project)
+
+    if conda env list | grep -E "^${wd}\s+" > /dev/null; then
+        conda-venv-activate
+    else
+        conda-venv-create "${py_ver}"
+    fi
+    command -v poetry > /dev/null
+    python --version
+}
+
+conda-pipenv () {
+    conda-venv "$1"
+    command -v pipenv > /dev/null | python -m pip install -U pipenv
+    pipenv --python="$(conda run which python)" --site-packages
+}
+
+conda-install () {
+    if ! command -v conda > /dev/null; then
+        # Support OSX and Linux - a Windows user can add support for it later
+        OS=$(uname)
+        if [ "$OS" == "Darwin" ]; then
+            installer='https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh'
+        elif [ "$OS" == "Linux" ]; then
+            installer='https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh'
+        fi
+        install_script="/tmp/$(basename $installer)"
+        wget --quiet $installer -O "$install_script"
+        /bin/bash "$install_script"
+    fi
+}

init.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+export CONDA_ENV="aiobotocore"
+
+# shellcheck disable=SC1091
+source ./conda_venv.sh
+
+if ! command -v conda > /dev/null; then
+    conda-install
+fi
+
+# Assume conda is already installed correctly and
+# source the conda config file from the usual paths
+# shellcheck disable=SC1091,SC1090
+source /etc/profile.d/conda.sh || \
+    source /opt/conda/etc/profile.d/conda.sh || \
+    source ~/miniconda3/etc/profile.d/conda.sh || \
+    source ~/opt/anaconda3/etc/profile.d/conda.sh
+
+conda-venv 3.6
+command -v python
+python --version
+python -m pip install --upgrade pip
+
+if ! command -v poetry > /dev/null; then
+  curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python
+fi
+
+# install dev-deps before the poetry project installation, so
+# that the project dependencies can override the dev-deps.
+if [ -f requirements.dev ]; then
+  poetry run python -m pip install -r requirements.dev
+  poetry run pre-commit install
+  # pip does not fully resolve conflicts, it just warns about them;
+  # use pip check to identify conflicts; some of them may not be
+  # important but others may need to be fixed somehow
+  poetry run python -m pip check
+fi
+
+# for development, install all the extras
+poetry install -v --no-interaction --extras all
+
+echo "To activate environment, execute 'conda activate ${CONDA_ENV}'"