[WIP] move to poetry (#831)

* [WIP] move to poetry

* [WIP] verbose in CI

* Fix pandas bug

* Bypass faulty type checking for pymysql connection

* Update dependencies

* Changes to test against CodeBuild unit tests and distribution build

* Remove editable install for docs

* Fix typing

Co-authored-by: Abdel Jaidi <jaidisido@gmail.com>

.bumpversion.cfg

@@ -4,6 +4,10 @@ commit = False
 tag = False
 tag_name = {new_version}
 
+[bumpversion:file:pyproject.toml]
+
+[bumpversion:file:test_infra/pyproject.toml]
+
 [bumpversion:file:README.md]
 
 [bumpversion:file:CONTRIBUTING_COMMON_ERRORS.md]

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+max-line-length = 120
+extend-ignore = E203, W503
+exclude = .git,__pycache__,docs/source/conf.py,old,build,dist,.venv,.venv2,.tox,dev,.env,.coverage

.github/workflows/minimal-tests.yml

@@ -29,7 +29,9 @@ jobs:
       - name: Install Requirements
         run: |
           python -m pip install --upgrade pip
-          pip install -U -r requirements-dev.txt
+          python -m pip install poetry==1.1.7
+          poetry config virtualenvs.create false --local
+          poetry install --extras "sqlserver" -vvv
       - name: Test Metadata
         run: pytest tests/test_metadata.py
       - name: Test Session

.github/workflows/static-checking.yml

@@ -27,7 +27,9 @@ jobs:
       - name: Install Requirements
         run: |
           python -m pip install --upgrade pip
-          pip install -U -r requirements-dev.txt
+          python -m pip install poetry==1.1.7
+          poetry config virtualenvs.create false --local
+          poetry install --extras "sqlserver" -vvv
       - name: mypy check
         run: mypy --install-types --non-interactive awswrangler
       - name: Flake8 Lint

.gitignore

@@ -26,6 +26,10 @@ share/python-wheels/
 *.egg
 MANIFEST
 
+# poetry
+poetry.toml
+envs.toml
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.

CONTRIBUTING.md

@@ -92,7 +92,7 @@ You can choose from three different environments to test your fixes/changes, bas
 ### Mocked test environment
 
 * Pick up a Linux or MacOS.
-* Install Python 3.7, 3.8 or 3.9
+* Install Python 3.7, 3.8 or 3.9 with [poetry](https://github.com/python-poetry/poetry) for package management
 * Fork the AWS Data Wrangler repository and clone that into your development environment
 * Go to the project's directory create a Python's virtual environment for the project
 
@@ -104,7 +104,7 @@ or
 
 * Install dependencies:
 
-``pip install -r requirements-dev.txt``
+``poetry install --extras "sqlserver"``
 
 * Run the validation script:
 
@@ -123,7 +123,7 @@ or
 **DISCLAIMER**: Make sure you know what you are doing. These steps will charge some services on your AWS account and require a minimum security skill to keep your environment safe.
 
 * Pick up a Linux or MacOS.
-* Install Python 3.7, 3.8 or 3.9
+* Install Python 3.7, 3.8 or 3.9 with [poetry](https://github.com/python-poetry/poetry) for package management
 * Fork the AWS Data Wrangler repository and clone that into your development environment
 * Go to the project's directory create a Python's virtual environment for the project
 
@@ -135,15 +135,15 @@ or
 
 * Install dependencies:
 
-``pip install -r requirements-dev.txt``
+``poetry install --extras "sqlserver"``
 
 * Go to the ``test_infra`` directory
 
 ``cd test_infra``
 
 * Install CDK dependencies:
 
-``pip install -r requirements.txt``
+``poetry install``
 
 * [OPTIONAL] Set AWS_DEFAULT_REGION to define the region the Data Lake Test environment will deploy into. You may want to choose a region which you don't currently use:
 
@@ -184,25 +184,25 @@ or
 **DISCLAIMER**: This environment contains Aurora MySQL, Aurora PostgreSQL and Redshift (single-node) clusters which will incur cost while running.
 
 * Pick up a Linux or MacOS.
-* Install Python 3.7, 3.8 or 3.9
+* Install Python 3.7, 3.8 or 3.9 with [poetry](https://github.com/python-poetry/poetry) for package management
 * Fork the AWS Data Wrangler repository and clone that into your development environment
 * Go to the project's directory create a Python's virtual environment for the project
 
 `python -m venv .venv && source .venv/bin/activate`
 
 * Then run the command bellow to install all dependencies:
 
-``pip install -r requirements-dev.txt``
+``poetry install --extras "sqlserver"``
 
 * Go to the ``test_infra`` directory
 
 ``cd test_infra``
 
 * Install CDK dependencies:
 
-``pip install -r requirements.txt``
+``poetry install``
 
-* [OPTIONAL] Set AWS_DEFAULT_REGION to define the region the Full Test envrioment will deploy into. You may want to choose a region which you don't currently use:
+* [OPTIONAL] Set AWS_DEFAULT_REGION to define the region the Full Test environment will deploy into. You may want to choose a region which you don't currently use:
 
 ``export AWS_DEFAULT_REGION=ap-northeast-1``
 

awswrangler/_utils.py

@@ -227,7 +227,7 @@ def chunkify(lst: List[Any], num_chunks: int = 1, max_length: Optional[int] = No
     if not lst:
         return []
     n: int = num_chunks if max_length is None else int(math.ceil((float(len(lst)) / float(max_length))))
-    np_chunks = np.array_split(lst, n)  # type: ignore
+    np_chunks = np.array_split(lst, n)
     return [arr.tolist() for arr in np_chunks if len(arr) > 0]
 
 

building/build-wheel.sh

@@ -2,6 +2,5 @@
 set -ex
 
 pushd ..
-rm -rf *.egg-info build dist/*.whl
-python setup.py bdist_wheel
-rm -rf *.egg-info build
+rm -rf dist/*.whl
+poetry build -f wheel

building/lambda/Dockerfile

@@ -12,17 +12,13 @@ RUN yum install -y \
     ninja-build \
     ${py_dev}
 
-RUN pip3 install --upgrade pip six cython cmake hypothesis
+RUN pip3 install --upgrade pip six cython cmake hypothesis poetry==1.1.7
 
-ADD requirements.txt /root/
-RUN pip3 install -r /root/requirements.txt
+WORKDIR /root
 
-ADD requirements-dev.txt /root/
-# Removing "-e ." installation
-RUN head -n -3 /root/requirements-dev.txt > /root/temp.txt
-RUN mv /root/temp.txt /root/requirements-dev.txt
-RUN pip3 install -r /root/requirements-dev.txt
+COPY pyproject.toml poetry.lock ./
+RUN poetry config virtualenvs.create false --local && poetry install --no-root
 
-RUN rm -rf /root/requirements*
+RUN rm -f pyproject.toml poetry.lock
 
 ENTRYPOINT ["/bin/sh"]

building/lambda/build-docker-images.sh

@@ -1,8 +1,8 @@
 #!/usr/bin/env bash
 set -ex
 
-cp ../../requirements.txt .
-cp ../../requirements-dev.txt .
+cp ../../pyproject.toml .
+cp ../../poetry.lock .
 
 # Python 3.6
 docker build \
@@ -28,4 +28,4 @@ docker build \
   --build-arg py_dev=python38-devel \
   .
 
-rm -rf requirements*
+rm -rf pyproject.toml poetry.lock

building/publish.sh

@@ -2,9 +2,6 @@
 set -ex
 
 pushd ..
-rm -fr build dist .egg awswrangler.egg-info
-python3.6 setup.py bdist_egg
-python3.6 setup.py bdist_wheel
-python3.6 setup.py sdist
-twine upload dist/*
-rm -fr build dist .egg awswrangler.egg-info
+rm -fr dist
+poetry publish --build
+rm -fr dist

docs/environment.yml

@@ -11,4 +11,4 @@ dependencies:
     - sphinx==4.0.3
     - sphinx_bootstrap_theme
     - IPython
-    - -e ..
+    - ..