Merge b2ec6e4 into 1707e20

.github/workflows/continuous-integration.yml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       max-parallel: 6
       matrix:
-        python-version: [ 3.7, 3.8, 3.9]
+        python-version: [ 3.6, 3.7, 3.8, 3.9]
         os: [ubuntu-latest, macOS-latest]
 
     steps:
@@ -25,6 +25,10 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      uses: abatilo/actions-poetry@v2.0.0
+      with:
+        poetry-version: 1.1.5
     - name: Install Protoc
       uses: arduino/setup-protoc@master
       with:
@@ -37,12 +41,9 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install tox tox-gh-actions pytest pytest-cov coverage  
-    - name: Test with tox
-      run: |
-        make test-all
+      run: make install
+    - name: Run tests
+      run: make test-system-python
     - name: Coveralls Parallel
       if: ${{matrix.os == 'ubuntu-latest'}}
       env:

.github/workflows/push_release.yml

@@ -17,13 +17,11 @@ jobs:
       with:
         python-version: '3.x'
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
+      run: make install
     - name: Build and publish
       env:
         TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
         TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
       run: |
-        python setup.py sdist bdist_wheel
+        make dist
         twine upload dist/*

.github/workflows/test-notebook.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
-    
+
     strategy:
       fail-fast: false
       max-parallel: 6
@@ -25,6 +25,10 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      uses: abatilo/actions-poetry@v2.0.0
+      with:
+        poetry-version: 1.1.5
     - name: Install Protoc
       uses: arduino/setup-protoc@master
       with:
@@ -37,10 +41,7 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements-test.txt
+      run: make install
     - name: Run notebook test
-      run: |
-        pytest --no-cov test_notebooks/notebook_tests.py
-    
+      run: make test-notebooks
+

.gitignore

@@ -143,4 +143,9 @@ cython_debug/
 
 
 #MacOS
-.DS_Store
+.DS_Store
+
+# Misc
+/output # default location of the whylogs binary output
+
+requirements.txt

.travis.yml

@@ -14,7 +14,7 @@ matrix:
     - env: DISTRIB="conda" PYTHON_VERSION="3.7" COVERAGE="false"
 install:
   - source tests/travis_install.sh
-  - pip install -r requirements.txt
+  - make install
   # ^  DEPRECATION WARNING:
   # The automatic creation of a `requirements.txt` file is deprecated.
   # See `Dependency Management` in the docs for other options.

DEVELOPMENT.md

@@ -4,82 +4,48 @@ Please take a look at this doc before contributing to whylogs python.
 
 ## Code format
 
-We use `flake8` for linting. To run flake8 lint verification:
-```
-make lint
+The following run the linter (flake8) and formatter (black). It fails if it finds issues but they won't be automatically resolved.
+
+```bash
+make lint format
 ```
 
-We use [black](https://pypi.org/project/black/) to format all our code.  Before submitting a PR you'll need to format the code by running (from the repo root dir):
-(note: this is currently disabled). Black is configured by `pyproject.toml`
+To automatically resolve issues, run
 
-```
-black .
+```bash
+make format-fix
 ```
 
 ## Development Environment
 
-1. It's recommended that you use [miniconda](https://docs.conda.io/en/latest/miniconda.html) to develop.
-
-2. Install [tox](https://tox.readthedocs.io/en/latest/) and [https://flake8.pycqa.org/en/latest/](flake8) and [black](https://black.readthedocs.io/en/stable/)
-    ```
-    # note that we are install these tools globally
-    pip install tox --user
-    pip install flake8 --user
-    pip install black --user
-    ```
-3. Clone the repo
+You'll need to install poetry in order to install dependencies using the lock file in this project. Follow [their docs](https://python-poetry.org/docs/) to get it set up.
 
-4. Clean potential previous 
-
-```
-make clean
 ```
+git clone https://github.com/whylabs/whylogs-java.git
+git submodule update --init --recursive
 
-5. Update all the submodules (to get the protobuf definitions): 
-
-    ```
-    git submodule update --init --recursive
-    ```
-
-6. Create a new conda environment for whylogs development. We need Python 3.7
- (though whylogs target multiple Python versions via `tox`):
-
-    ```
-    conda create --name=whylogs-dev python=3.7
-    conda activate whylogs-dev
-    ```
-
-7. Install dependencies
-
-    ```
-    conda install pip
-    pip install -r requirements-dev.txt
-    ```
+# Use poetry to install dependencies
+make install
 
-8. Install whylogs in editable mode to the current python environment
+# Build the protobuf source and the distribution tar and wheel
+make
 
-    ```
-    make develop
-    ```
-
-9. (optional) Build and serve documentation
-
-    ```
-    make docs
-    make servedocs
-    ```
+# For building and viewing the docs if you're working on those
+make docs
+```
 
+## Managing Dependencies
 
+TODO
 
 
 ## Testing
-
 To run tests using the current Python environment:
+
 ```
 make test
 ```
 
-
 ### Coverage 
 
 Coverage can be checked with 
@@ -90,7 +56,9 @@ make coverage
 
 ### Testing CI locally
 
-you can run local github actions on the ubuntu using https://github.com/nektos/act. Currently you need to build a latest docker image for ubuntu using the following dockerfile
+TODO make this a real docker file then
+
+you can run local github actions on ubuntu using [act](https://github.com/nektos/act). Currently, you need to build the latest docker image for ubuntu using the following dockerfile
 
 ```dockerfile
 FROM ubuntu:20.04