Try conda/poetry for dev-env

.pre-commit-config.yaml

@@ -0,0 +1,20 @@
+repos:
+    - repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v3.1.0
+      hooks:
+        - id: check-yaml
+          args: [--unsafe]
+          exclude: '.*templates.*.yaml'
+        - id: end-of-file-fixer
+          exclude: 'CHANGELOG.md'
+        - id: trailing-whitespace
+          exclude: 'CHANGELOG.md'
+        - id: check-ast
+    - repo: https://github.com/gruntwork-io/pre-commit
+      rev: v0.1.9
+      hooks:
+        - id: shellcheck
+    - repo: https://github.com/psf/black
+      rev: 19.10b0
+      hooks:
+        - id: black

CONTRIBUTING.rst

@@ -17,12 +17,14 @@ Create virtualenv with at least python3.5 (older version are not supported).
 For example using *virtualenvwrapper* commands could look like::
 
    $ cd aiobotocore
-   $ mkvirtualenv --python=`which python3.5` aiobotocore
-
-
-After that please install libraries required for development::
-
-    $ pipenv sync --dev
+   $ make init
+
+The virtualenv is provided by a conda installation and the project
+dependencies are installed by poetry.  See the `Makefile` and the
+`init.sh` script for details.  The `make init` script is designed
+to be run only when a clean virtualenv is created; after that, the
+virtualenv is usually only activated and packages are managed using
+poetry.
 
 Congratulations, you are ready to run the test suite::
 

Makefile

@@ -2,28 +2,35 @@
 
 FLAGS=
 
+LIB=aiobotocore
+
+# the install uses conda to provide a python 3.6 base environment;
+# the init.sh script uses a reliable conda environment to do so.
+init: poetry
+	./init.sh
+
 flake: checkrst
-	pipenv run python3 -m flake8 --format=abspath
+	@poetry run python3 -m flake8 --format=abspath
 
 test: flake
-	pipenv run python3 -Wd -m pytest -s -vv $(FLAGS) ./tests/
+	@poetry run python3 -Wd -m pytest -s -vv $(FLAGS) ./tests/
 
 vtest:
-	pipenv run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -s -vv $(FLAGS) ./tests/
+	@poetry run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -s -vv $(FLAGS) ./tests/
 
 checkrst:
-	pipenv run python3 setup.py check -rms
+	# TODO: remove the setup.py file or make it work with a poetry build?
+	@poetry run python3 setup.py check -rms
 
 cov cover coverage: flake
-	pipenv run python3 -Wd -m pytest -s -vv --cov-report term --cov-report html --cov aiobotocore ./tests
+	@poetry run python3 -Wd -m pytest -s -vv --cov-report term --cov-report html --cov aiobotocore ./tests
 	@echo "open file://`pwd`/htmlcov/index.html"
 
 # BOTO_CONFIG solves https://github.com/travis-ci/travis-ci/issues/7940
 mototest:
-	BOTO_CONFIG=/dev/null pipenv run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -vv -m moto -n auto --cov-report term --cov-report html --cov aiobotocore tests
+	BOTO_CONFIG=/dev/null poetry run python3 -Wd -X tracemalloc=5 -X faulthandler -m pytest -vv -m moto -n auto --cov-report term --cov-report html --cov aiobotocore tests
 	@echo "open file://`pwd`/htmlcov/index.html"
 
-
 clean:
 	rm -rf `find . -name __pycache__`
 	rm -f `find . -type f -name '*.py[co]' `
@@ -43,4 +50,26 @@ doc:
 	make -C docs html
 	@echo "open file://`pwd`/docs/_build/html/index.html"
 
+typehint: clean
+	@poetry run mypy --follow-imports=skip $(LIB) tests
+
+package: clean
+	@poetry check
+	@poetry build
+
+package-check: package
+	@poetry run twine check dist/*
+
+poetry:
+	@if ! command -v poetry > /dev/null; then \
+		curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py -o /tmp/get-poetry.py; \
+		python /tmp/get-poetry.py; \
+	fi
+
+poetry-export:
+	poetry export --without-hashes -f requirements.txt -o requirements.txt
+	sed -i -e 's/^-e //g' requirements.txt
+
+
+.PHONY: init typehint package package-check poetry poetry-export
 .PHONY: all flake test vtest cov clean doc

conda_venv.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+# Copyright 2019-2020 Darren Weber
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# https://python-release-cycle.glitch.me/
+
+# Source this file from ~/.bashrc or similar shell-init, such
+# as copy the file to /etc/profile.d/
+
+
+# Enable this to create conda env for each python version
+if false; then
+    conda create -y -n py3.6 python=3.6
+    conda create -y -n py3.7 python=3.7
+    conda create -y -n py3.8 python=3.8
+fi
+
+# CONDA PYTHON VERSIONS
+alias py36='conda deactivate; conda activate py3.6'
+alias py37='conda deactivate; conda activate py3.7'
+alias py38='conda deactivate; conda activate py3.8'
+
+conda-project () {
+    # The project name is defined by CONDA_ENV or the current working directory
+    project=${CONDA_ENV:-$(pwd)}
+    basename "${project}"
+}
+
+conda-venv-activate () {
+    # try to activate a conda environment with the name of
+    # the current directory (often this is a project name).
+    wd=$(conda-project)
+    conda deactivate
+    conda activate "$wd"
+}
+
+conda-venv-create () {
+    # create and activate a conda environment with the name
+    # of the current directory (often this is a project name).
+    py_ver="${1:-3.6}"
+    wd=$(conda-project)
+    conda deactivate
+
+    conda create -n "$wd" python="${py_ver}" \
+      --channel conda-forge --override-channels \
+    && conda activate "$wd" \
+    && conda config --env --add channels conda-forge \
+    && conda config --env --set channel_priority strict
+}
+
+conda-venv-remove () {
+    # try to activate a conda environment with the name of
+    # the current directory (often this is a project name).
+    wd=$(conda-project)
+    conda deactivate
+    conda env remove -n "$wd"
+}
+
+conda-venv () {
+    # create and activate a conda environment with the name
+    # of the current directory (often this is a project name).
+    py_ver="${1:-3.6}"
+    wd=$(conda-project)
+
+    if conda env list | grep -E "^${wd}\s+" > /dev/null; then
+        conda-venv-activate
+    else
+        conda-venv-create "${py_ver}"
+    fi
+    command -v poetry > /dev/null
+    python --version
+}
+
+conda-pipenv () {
+    conda-venv "$1"
+    command -v pipenv > /dev/null | python -m pip install -U pipenv
+    pipenv --python="$(conda run which python)" --site-packages
+}
+
+conda-install () {
+    if ! command -v conda > /dev/null; then
+        # Support OSX and Linux - a Windows user can add support for it later
+        OS=$(uname)
+        if [ "$OS" == "Darwin" ]; then
+            installer='https://repo.anaconda.com/miniconda/Miniconda3-latest-MacOSX-x86_64.sh'
+        elif [ "$OS" == "Linux" ]; then
+            installer='https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh'
+        fi
+        install_script="/tmp/$(basename $installer)"
+        wget --quiet $installer -O "$install_script"
+        /bin/bash "$install_script"
+    fi
+}

init.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+export CONDA_ENV="aiobotocore"
+
+# shellcheck disable=SC1091
+source ./conda_venv.sh
+
+conda-install
+
+# Assume conda is already installed correctly and
+# source the conda config file from the usual paths
+# shellcheck disable=SC1091,SC1090
+source ~/miniconda3/etc/profile.d/conda.sh || \
+  source /etc/profile.d/conda.sh || \
+  source ~/opt/anaconda3/etc/profile.d/conda.sh
+
+conda-venv 3.6
+command -v python
+python --version
+python -m pip install --upgrade pip
+
+if ! command -v poetry > /dev/null; then
+  curl -sSL https://raw.githubusercontent.com/sdispater/poetry/master/get-poetry.py | python
+fi
+
+# install dev-deps before the poetry project installation, so
+# that the project dependencies can override the dev-deps.
+if [ -f requirements.dev ]; then
+  poetry run python -m pip install -r requirements.dev
+  poetry run pre-commit install
+  # pip does not fully resolve conflicts, it just warns about them;
+  # use pip check to identify conflicts; some of them may not be
+  # important but others may need to be fixed somehow
+  poetry run python -m pip check
+fi
+
+# for development, install all the extras
+poetry install -v --no-interaction --extras all
+
+echo "To activate environment, execute 'conda activate ${CONDA_ENV}'"