Merge a3812b1 into 1707e20

.github/workflows/continuous-integration.yml

@@ -25,6 +25,10 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      uses: abatilo/actions-poetry@v2.0.0
+      with:
+        poetry-version: 1.1.5
     - name: Install Protoc
       uses: arduino/setup-protoc@master
       with:
@@ -37,12 +41,9 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install tox tox-gh-actions pytest pytest-cov coverage  
-    - name: Test with tox
-      run: |
-        make test-all
+      run: make install
+    - name: Run tests
+      run: make test-system-python
     - name: Coveralls Parallel
       if: ${{matrix.os == 'ubuntu-latest'}}
       env:

.github/workflows/push_release.yml

@@ -17,13 +17,11 @@ jobs:
       with:
         python-version: '3.x'
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
+      run: make install
     - name: Build and publish
       env:
         TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
         TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
       run: |
-        python setup.py sdist bdist_wheel
+        make dist
         twine upload dist/*

.github/workflows/test-notebook.yml

@@ -9,7 +9,7 @@ on:
 jobs:
   test:
     runs-on: ${{ matrix.os }}
-    
+
     strategy:
       fail-fast: false
       max-parallel: 6
@@ -25,6 +25,10 @@ jobs:
       uses: actions/setup-python@v2
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Install Poetry
+      uses: abatilo/actions-poetry@v2.0.0
+      with:
+        poetry-version: 1.1.5
     - name: Install Protoc
       uses: arduino/setup-protoc@master
       with:
@@ -37,10 +41,7 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install -r requirements-test.txt
+      run: make install
     - name: Run notebook test
-      run: |
-        pytest --no-cov test_notebooks/notebook_tests.py
-    
+      run: make test-notebooks
+

.gitignore

@@ -143,4 +143,9 @@ cython_debug/
 
 
 #MacOS
-.DS_Store
+.DS_Store
+
+# Misc
+/output # default location of the whylogs binary output
+
+requirements.txt

.travis.yml

@@ -14,7 +14,7 @@ matrix:
     - env: DISTRIB="conda" PYTHON_VERSION="3.7" COVERAGE="false"
 install:
   - source tests/travis_install.sh
-  - pip install -r requirements.txt
+  - make install
   # ^  DEPRECATION WARNING:
   # The automatic creation of a `requirements.txt` file is deprecated.
   # See `Dependency Management` in the docs for other options.

Makefile

@@ -1,94 +1,145 @@
-.PHONY: clean clean-test clean-pyc clean-build docs help
-.DEFAULT_GOAL := help
+src.python := $(shell find ./src -type f -name "*.py")
+tst.python := $(shell find ./tests -type f -name "*.py")
+src.python.pyc := $(shell find ./src -type f -name "*.pyc")
+src.proto.dir := ./proto/src
+src.proto := $(shell find $(src.proto.dir) -type f -name "*.proto")
 
-define BROWSER_PYSCRIPT
-import os, webbrowser, sys
+dist.dir := dist
+egg.dir := .eggs
+build.dir := build
+# This isn't exactly true but its the only thing that we easily know the name of at this point. Its a good proxy for
+# the wheel since its created along with it.
+build.wheel := $(dist.dir)/whylogs-0.4.5.dev1.tar.gz
+build.proto.dir := src/whylogs/proto
+build.proto := $(patsubst $(src.proto.dir)/%.proto,$(build.proto.dir)/%_pb2.py,$(src.proto))
 
-from urllib.request import pathname2url
+default: dist
 
-webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
-endef
-export BROWSER_PYSCRIPT
+release:
+	# Run format checker
+	# Run dist
 
-define PRINT_HELP_PYSCRIPT
-import re, sys
+github:
+	# TODO
 
-for line in sys.stdin:
-	match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line)
-	if match:
-		target, help = match.groups()
-		print("%-20s %s" % (target, help))
-endef
-export PRINT_HELP_PYSCRIPT
+.PHONY: dist clean clean-test help format lint test install coverage docs default proto test-notebooks github release test-system-python
 
-BROWSER := python -c "$$BROWSER_PYSCRIPT"
+ifeq (, $(shell which poetry))
+	$(error "Can't find poetry on the path. Install it at https://python-poetry.org/docs.")
+endif
 
 help:
 	@python -c "$$PRINT_HELP_PYSCRIPT" < $(MAKEFILE_LIST)
 
-clean: clean-build clean-pyc clean-test ## remove all build, test, coverage and Python artifacts
-
-clean-build: ## remove build artifacts
-	rm -fr build/
-	rm -fr dist/
-	rm -fr .eggs/
-	rm -fr src/whylogs/proto/*pb2.py
-	find . -name '*.egg-info' -exec rm -fr {} +
-	find . -name '*.egg' -exec rm -f {} +
-
-clean-pyc: ## remove Python file artifacts
-	find . -name '*.pyc' -exec rm -f {} +
-	find . -name '*.pyo' -exec rm -f {} +
-	find . -name '*~' -exec rm -f {} +
-	find . -name '__pycache__' -exec rm -fr {} +
+clean: clean-test ## Remove all build artifacts
+	rm -rf $(dist.dir)
+	rm -rf $(build.dir)
+	rm -f $(src.python.pyc)
+	rm -rf $(egg.dir)
+	rm -rf $(build.proto)
+	rm -f $(build.proto)
+	rm -f requirements.txt
 
-clean-test: ## remove test and coverage artifacts
+clean-test: ## Remove test and coverage artifacts
 	rm -fr .tox/
 	rm -f .coverage
 	rm -fr htmlcov/
 	rm -fr .pytest_cache
 
+dist: $(build.wheel) ## Create distribution tarballs and wheels
+
+$(build.wheel): $(src.python) $(build.proto)
+	@$(call i, Generating distribution files)
+	poetry build
+	@$(call i, Distribution files created)
+	@find dist -type f
+
+proto: $(build.proto)
+
+requirements.txt:
+	@$(call i, Generating a requirements.txt file from poetry)
+	poetry export -f requirements.txt --output requirements.txt --dev
+
+$(build.proto): $(src.proto)
+	@$(call i, Generating python source for protobuf)
+	protoc -I $(src.proto.dir) --python_out=$(build.proto.dir) $(src.proto)
+	poetry run 2to3 --nobackups --write ./src/whylogs/proto/
+
 lint: ## check style with flake8
-	tox -e flake8
+	@$(call i, Running the linter)
+	poetry run tox -e flake8
+
+format: ## format source code with black
+	@$(call i, Running the formatter)
+	poetry run black .
 
-test: build-proto ## run tests quickly with the default Python
-	pytest
+test: dist ## run tests with pytest
+	@$(call i, Running tests)
+	poetry run pytest
 
-test-all: build-proto ## run tests on every Python version with tox
-	tox
+test-system-python: dist ## Run tests using the system `python` instead of the locally declared poetry python
+	@$(call i, Running tests using the globally installed python)
+	python -m poetry run python --version
+	python -m poetry run pytest
 
-coverage: ## check code coverage quickly with the default Python
-	pytest --cov='src/.' tests/
-	python -m coverage report
+test-notebooks: ## Run tests for the notebooks
+	@$(call i, Running notebook tests)
+	poetry run pytest --no-cov test_notebooks/notebook_tests.py
 
-docs: build-proto ## generate Sphinx HTML documentation, including API docs
+install: ## install all dependencies with poetry
+	@$(call i, Installing dependencies)
+	poetry install
+
+coverage: ## generate test coverage reports
+	@$(call i, Generating test coverage)
+	poetry run pytest --cov='src/.' tests/
+	poetry run python -m coverage report
+
+docs: proto ## generate Sphinx HTML documentation, including API docs
+	@$(call i, Generating docs)
 	rm -f docs/whylogs.rst
 	rm -f docs/modules.rst
 	cd docs
 	$(MAKE) -C docs clean
 	$(MAKE) -C docs html
 	$(BROWSER) build/sphinx/html/index.html
 
-servedocs: docs ## compile the docs watching for changes
-	watchmedo shell-command -p '*.rst' -recursive -c '$(MAKE) -C docs' -D .
 
-release: dist ## package and upload a release
-	twine upload dist/*
+define BROWSER_PYSCRIPT
+import os, webbrowser, sys
 
-dist: clean-build build-proto ## builds source and wheel package
-	python setup.py build
-	python setup.py sdist
-	python setup.py bdist_wheel
-	ls -l dist
+from urllib.request import pathname2url
 
-install: build-proto clean ## install the package to the active Python's site-packages
-	python setup.py install
+webbrowser.open("file://" + pathname2url(os.path.abspath(sys.argv[1])))
+endef
+export BROWSER_PYSCRIPT
+BROWSER := python -c "$$BROWSER_PYSCRIPT"
 
-build-proto:
-	python setup.py proto
+define PRINT_HELP_PYSCRIPT
+import re, sys
 
-build: build-proto lint
-	python setup.py build
+for line in sys.stdin:
+	match = re.match(r'^([a-zA-Z_-]+):.*?## (.*)$$', line)
+	if match:
+		target, help = match.groups()
+		print("%-20s %s" % (target, help))
+endef
+export PRINT_HELP_PYSCRIPT
 
-develop: build-proto
-	python setup.py develop
+define i
+echo
+echo "[INFO] $1"
+echo
+endef
+
+define w
+echo
+echo "[WARN] $1"
+echo
+endef
+
+define e
+echo
+echo "[ERROR] $1"
+echo
+endef