chore: tidy up makefile (DEV-1166) (#223)

.github/workflows/daily-test.yml

@@ -19,8 +19,7 @@ jobs:
                     python-version: 3.9
             -   name: Install dependencies
                 run: |
-                    make upgrade-dist-tools
                     make install-requirements
                     make install
             -   name: Run e2e tests
-                run: make test-end-to-end
+                run: make test-end-to-end-ci

.github/workflows/release.yml

@@ -29,10 +29,9 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install -r requirements.txt
+          make install-requirements
       # release new version to PyPI
       - run: |
-          make upgrade-dist-tools
           make dist
           make upload
 

.github/workflows/test.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        target: [ 'test-end-to-end', 'test-unittests', 'docs-build' ]
+        target: [ 'test-end-to-end-ci', 'test-unittests', 'docs-build' ]
     steps:
       - name: Checkout source
         uses: actions/checkout@v2
@@ -23,7 +23,6 @@ jobs:
           python-version: 3.9
       - name: Install dependencies
         run: |
-            make upgrade-dist-tools
             make install-requirements
             make install
       - name: Run tests

Makefile

@@ -3,21 +3,23 @@
 THIS_FILE := $(lastword $(MAKEFILE_LIST))
 CURRENT_DIR := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
 
-#################################
+############################
 # Make targets for dsp-tools
-#################################
-
-.PHONY: clone-dsp-repo
-clone-dsp-repo: ## clone the dsp-api git repository
-	@git clone --branch main --single-branch --depth 1 https://github.com/dasch-swiss/dsp-api.git $(CURRENT_DIR)/.tmp/dsp-stack
+############################
 
 .PHONY: dsp-stack
-dsp-stack: ## run the dsp-stack (deletes existing volumes first)
-	$(MAKE) -C $(CURRENT_DIR)/.tmp/dsp-stack env-file
-	$(MAKE) -C $(CURRENT_DIR)/.tmp/dsp-stack stack-down-delete-volumes
-	$(MAKE) -C $(CURRENT_DIR)/.tmp/dsp-stack init-db-test
-	$(MAKE) -C $(CURRENT_DIR)/.tmp/dsp-stack stack-up
-	$(MAKE) -C $(CURRENT_DIR)/.tmp/dsp-stack stack-logs-api-no-follow
+dsp-stack: ## clone the dsp-api git repository and run the dsp-stack
+	@mkdir -p .tmp
+	@git clone --branch main --single-branch --depth 1 https://github.com/dasch-swiss/dsp-api.git .tmp/dsp-stack
+	$(MAKE) -C .tmp/dsp-stack env-file
+	$(MAKE) -C .tmp/dsp-stack init-db-test
+	$(MAKE) -C .tmp/dsp-stack stack-up
+	$(MAKE) -C .tmp/dsp-stack stack-logs-api-no-follow
+
+.PHONY: stack-down
+stack-down: ## stop dsp-stack and remove the cloned dsp-api repository
+	$(MAKE) -C .tmp/dsp-stack stack-down-delete-volumes
+	@test -x .tmp && rm -rf .tmp
 
 .PHONY: dist
 dist: ## generate distribution package
@@ -27,21 +29,13 @@ dist: ## generate distribution package
 upload: ## upload distribution package to PyPI
 	python3 -m twine upload dist/*
 
-.PHONY: upgrade-dist-tools
-upgrade-dist-tools: ## upgrade packages necessary for testing, building, packaging and uploading to PyPI
-	python3 -m pip install --upgrade pip setuptools wheel twine pytest mkdocs
-
 .PHONY: docs-build
 docs-build: ## build docs into the local 'site' folder
 	mkdocs build
 
 .PHONY: docs-serve
 docs-serve: ## serve docs for local viewing
-	mkdocs serve
-
-.PHONY: docs-publish
-docs-publish: ## build and publish docs to GitHub Pages
-	mkdocs gh-deploy
+	mkdocs serve --dev-addr=0.0.0.0:7979
 
 .PHONY: install-requirements
 install-requirements: ## install requirements
@@ -56,44 +50,47 @@ install: ## install from source (runs setup.py)
 	pip3 install -e .
 
 .PHONY: test
-test: clean local-tmp clone-dsp-repo dsp-stack ## run all tests
+test: dsp-stack ## run all tests
+	pytest test/
+	$(MAKE) stack-down
+
+.PHONY: test-no-stack
+test-no-stack: ## run tests without starting the stack (if a dsp-stack is already running)
 	pytest test/
 
 .PHONY: test-end-to-end
-test-end-to-end: clean local-tmp clone-dsp-repo dsp-stack ## run e2e tests
+test-end-to-end: dsp-stack ## run e2e tests
+	pytest test/e2e/
+	$(MAKE) stack-down
+
+.PHONY: test-end-to-end-ci
+test-end-to-end-ci: dsp-stack ## run e2e tests on GitHub CI, where it isn't possible nor necessary to remove .tmp
+	pytest test/e2e/
+
+.PHONY: test-end-to-end-no-stack
+test-end-to-end-no-stack: ## run e2e tests without starting the dsp-stack (if a dsp-stack is already running)
 	pytest test/e2e/
 
 .PHONY: test-unittests
 test-unittests: ## run unit tests
 	pytest test/unittests/
 
-.PHONY: local-tmp
-local-tmp: ## create local .tmp folder
-	@mkdir -p $(CURRENT_DIR)/.tmp
-
 .PHONY: clean
 clean: ## clean local project directories
-	@rm -rf $(CURRENT_DIR)/.tmp
-	@rm -rf dist/ build/ site/ knora.egg-info/
+	@rm -rf dist/ build/ site/ dsp_tools.egg-info/
 
 .PHONY: help
 help: ## show this help
 	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) | sort
 
-.PHONY: run
-run: ## create dist, install and run
-	$(MAKE) clean
-	$(MAKE) dist
-	$(MAKE) install
-	dsp-tools
-
 .PHONY: freeze-requirements
 freeze-requirements: ## update (dev-)requirements.txt and setup.py based on pipenv's Pipfile.lock
-	pipenv requirements > requirements.txt
+	pipenv run pipenv requirements > requirements.txt
 	sed -i '' 's/==/~=/g' requirements.txt
-	pipenv requirements --dev-only > dev-requirements.txt
+	pipenv run pipenv requirements --dev-only > dev-requirements.txt
 	sed -i '' 's/==/~=/g' dev-requirements.txt
 	pipenv run pipenv-setup sync
 	sed -i '' 's/==/~=/g' setup.py
+	autopep8 --global-config pyproject.toml --aggressive --in-place setup.py
 
 .DEFAULT_GOAL := help

Pipfile

@@ -6,20 +6,15 @@ name = "pypi"
 [packages]
 jsonpath-ng = "*"
 argparse = "*"
-rdflib = "*"
 lxml = "*"
-validators = "*"
 requests = "*"
 jsonschema = "*"
-click = "*"
-rfc3987 = "*"
 pystrict = "*"
 openpyxl = "*"
-pyparsing = "==2.4.7"
 networkx = "*"
 pandas = "*"
-xlrd = "*"
 regex = "*"
+xlrd = "*"
 
 [dev-packages]
 mkdocs = "*"
@@ -32,6 +27,7 @@ wheel = "*"
 pipenv-setup = "*"
 pytest = "*"
 types-requests = "*"
+twine = "*"
 
 [requires]
 python_version = "3"