feat: better makefile/tox

.gitignore

@@ -31,6 +31,7 @@ pip-log.txt
 .tox
 coverage.xml
 htmlcov/
+pii_report/
 
 # Virtual environments
 /venv/

Makefile

@@ -1,6 +1,9 @@
-.PHONY: clean clean_tox compile_translations coverage diff_cover docs dummy_translations \
-        extract_translations fake_translations help pii_check pull_translations push_translations \
-        quality requirements selfcheck test test-all upgrade validate install_transifex_client
+.PHONY: clean clean_tox compile_translations coverage docs dummy_translations \
+        extract_translations fake_translations help pull_translations push_translations \
+        quality requirements selfcheck test test-all upgrade install_transifex_client
+
+SRC_FILES_PROD = forum tests test_utils manage.py
+SRC_FILES = ${SRC_FILES_PROD} setup.py test_settings.py
 
 .DEFAULT_GOAL := help
 
@@ -32,35 +35,27 @@ docs: ## generate Sphinx HTML documentation, including API docs
 	tox -e docs
 	$(BROWSER)docs/_build/html/index.html
 
-mypy_check:
-	mypy forum/ tests/ test_utils/
-
-# Define PIP_COMPILE_OPTS=-v to get more information during make upgrade.
-PIP_COMPILE = pip-compile --upgrade $(PIP_COMPILE_OPTS)
-
-upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade
-upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
-	pip install -qr requirements/pip-tools.txt
+# Define PIP_COMPILE_OPTS="-v" to get more information during make compile-requirements.
+compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade
+compile-requirements: ## Re-compile *.in requirements to *.txt
+	pip install --quiet -r requirements/pip-tools.txt
 	# Make sure to compile files after any other files they include!
-	$(PIP_COMPILE) --allow-unsafe -o requirements/pip.txt requirements/pip.in
-	$(PIP_COMPILE) -o requirements/pip-tools.txt requirements/pip-tools.in
-	pip install -qr requirements/pip.txt
-	pip install -qr requirements/pip-tools.txt
-	$(PIP_COMPILE) -o requirements/base.txt requirements/base.in
-	$(PIP_COMPILE) -o requirements/test.txt requirements/test.in
-	$(PIP_COMPILE) -o requirements/doc.txt requirements/doc.in
-	$(PIP_COMPILE) -o requirements/quality.txt requirements/quality.in
-	$(PIP_COMPILE) -o requirements/ci.txt requirements/ci.in
-	$(PIP_COMPILE) -o requirements/dev.txt requirements/dev.in
+	pip-compile $(COMPILE_OPTS) --allow-unsafe requirements/pip.in
+	pip-compile $(COMPILE_OPTS) requirements/pip-tools.in
+	pip install --quiet -r requirements/pip.txt
+	pip install --quiet -r requirements/pip-tools.txt
+	pip-compile $(COMPILE_OPTS) requirements/base.in
+	pip-compile $(COMPILE_OPTS) requirements/test.in
+	pip-compile $(COMPILE_OPTS) requirements/doc.in
+	pip-compile $(COMPILE_OPTS) requirements/quality.in
+	pip-compile $(COMPILE_OPTS) requirements/ci.in
+	pip-compile $(COMPILE_OPTS) requirements/dev.in
 	# Let tox control the Django version for tests
 	sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp
 	mv requirements/test.tmp requirements/test.txt
 
-quality: ## check coding style with pycodestyle and pylint
-	tox -e quality
-
-pii_check: ## check for PII annotations on all Django models
-	tox -e pii_check
+upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
+	$(MAKE) compile-requirements COMPILE_OPTS="--upgrade"
 
 piptools: ## install pinned version of pip-compile and pip-sync
 	pip install -r requirements/pip.txt
@@ -69,17 +64,29 @@ piptools: ## install pinned version of pip-compile and pip-sync
 requirements: clean_tox piptools ## install development environment requirements
 	pip-sync -q requirements/dev.txt requirements/private.*
 
-test: clean ## run tests in the current virtualenv
+test-all: clean test test-quality test-pii selfcheck ## run all tests
+
+test: ## run unit tests
 	pytest
 
-diff_cover: test ## find diff lines that need test coverage
-	diff-cover coverage.xml
+test-quality: test-lint test-codestyle test-isort test-mypy ## run static coverage tests
 
-test-all: quality pii_check ## run tests on every supported Python/Django combination
-	tox
-	tox -e docs
+test-lint: ## run pylint
+	pylint ${SRC_FILES}
+
+test-codestyle: ## run pycodestyle, pydocstyle
+	pycodestyle ${SRC_FILES}
+	pydocstyle ${SRC_FILES}
+
+test-isort: ## run isort checks
+	isort --check-only --diff ${SRC_FILES}
+
+test-mypy: ## run type tests
+	mypy ${SRC_FILES_PROD}
 
-validate: quality pii_check test ## run tests and quality checks
+test-pii: export DJANGO_SETTINGS_MODULE=test_settings
+test-pii: ## # check for PII annotations on all Django models
+	 code_annotations django_find_annotations --config_file .pii_annotations.yml --lint --report --coverage
 
 selfcheck: ## check that the Makefile is well-formed
 	@echo "The Makefile is well-formed."

docs/testing.rst

@@ -3,38 +3,37 @@
 Testing
 #######
 
-forum has an assortment of test cases and code quality
-checks to catch potential problems during development.  To run them all in the
+This project has an assortment of test cases and code quality checks to catch potential problems during development. To run them all in the
 version of Python you chose for your virtualenv:
 
 .. code-block:: bash
 
-    $ make validate
+    $ make test-all
 
 To run just the unit tests:
 
 .. code-block:: bash
 
     $ make test
 
-To run just the unit tests and check diff coverage
+To run just the unit tests and check the difference in coverage with the master branch:
 
 .. code-block:: bash
 
-    $ make diff_cover
+    $ diff-cover --compare-branch=master coverage.xml
 
-To run just the code quality checks:
+To run just the code quality checks (including mypy checks):
 
 .. code-block:: bash
 
-    $ make quality
+    $ make test-quality
 
 To run the unit tests under every supported Python version and the code
 quality checks:
 
 .. code-block:: bash
 
-    $ make test-all
+    $ tox -e pii_check,quality,py,docs
 
 To generate and open an HTML report of how much of the code is covered by
 test cases:

requirements/base.txt

@@ -1,16 +1,11 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    make upgrade
 #
 asgiref==3.8.1
     # via django
-backports-zoneinfo==0.2.1 ; python_version < "3.9"
-    # via
-    #   -c requirements/constraints.txt
-    #   django
-    #   djangorestframework
 certifi==2024.7.4
     # via requests
 charset-normalizer==3.3.2

requirements/ci.in

@@ -3,12 +3,3 @@
 -c constraints.txt
 
 -r quality.txt            # Core and quality check dependencies
-
-tox                       # Virtualenv management for tests
-
-# mypy for typing checks
-mypy
-
-# add below 2 packages as mypy giving errors of missing stubs
-djangorestframework-stubs
-types-requests

requirements/ci.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    make upgrade
@@ -14,12 +14,6 @@ astroid==3.2.4
     #   -r requirements/quality.txt
     #   pylint
     #   pylint-celery
-backports-zoneinfo==0.2.1 ; python_version < "3.9"
-    # via
-    #   -c requirements/constraints.txt
-    #   -r requirements/quality.txt
-    #   django
-    #   djangorestframework
 cachetools==5.4.0
     # via
     #   -r requirements/quality.txt
@@ -74,13 +68,17 @@ django==4.2.15
     #   django-stubs-ext
     #   djangorestframework
 django-stubs==5.0.4
-    # via djangorestframework-stubs
+    # via
+    #   -r requirements/quality.txt
+    #   djangorestframework-stubs
 django-stubs-ext==5.0.4
-    # via django-stubs
+    # via
+    #   -r requirements/quality.txt
+    #   django-stubs
 djangorestframework==3.15.2
     # via -r requirements/quality.txt
 djangorestframework-stubs==3.15.0
-    # via -r requirements/ci.in
+    # via -r requirements/quality.txt
 dnspython==2.6.1
     # via
     #   -r requirements/quality.txt
@@ -123,9 +121,7 @@ mccabe==0.7.0
 mongomock==4.1.2
     # via -r requirements/quality.txt
 mypy==1.11.1
-    # via
-    #   -r requirements/ci.in
-    #   -r requirements/quality.txt
+    # via -r requirements/quality.txt
 mypy-extensions==1.0.0
     # via
     #   -r requirements/quality.txt
@@ -244,16 +240,15 @@ tomlkit==0.13.0
     #   -r requirements/quality.txt
     #   pylint
 tox==4.17.1
-    # via
-    #   -r requirements/ci.in
-    #   -r requirements/quality.txt
+    # via -r requirements/quality.txt
 types-pyyaml==6.0.12.20240808
     # via
+    #   -r requirements/quality.txt
     #   django-stubs
     #   djangorestframework-stubs
 types-requests==2.32.0.20240712
     # via
-    #   -r requirements/ci.in
+    #   -r requirements/quality.txt
     #   djangorestframework-stubs
 typing-extensions==4.12.2
     # via
@@ -264,7 +259,6 @@ typing-extensions==4.12.2
     #   django-stubs-ext
     #   djangorestframework-stubs
     #   mypy
-    #   pylint
 urllib3==2.2.2
     # via
     #   -r requirements/quality.txt

requirements/dev.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    make upgrade
@@ -16,13 +16,6 @@ astroid==3.2.4
     #   -r requirements/quality.txt
     #   pylint
     #   pylint-celery
-backports-zoneinfo==0.2.1 ; python_version < "3.9"
-    # via
-    #   -c requirements/constraints.txt
-    #   -r requirements/ci.txt
-    #   -r requirements/quality.txt
-    #   django
-    #   djangorestframework
 build==1.2.1
     # via
     #   -r requirements/pip-tools.txt
@@ -101,17 +94,21 @@ django==4.2.15
 django-stubs==5.0.4
     # via
     #   -r requirements/ci.txt
+    #   -r requirements/quality.txt
     #   djangorestframework-stubs
 django-stubs-ext==5.0.4
     # via
     #   -r requirements/ci.txt
+    #   -r requirements/quality.txt
     #   django-stubs
 djangorestframework==3.15.2
     # via
     #   -r requirements/ci.txt
     #   -r requirements/quality.txt
 djangorestframework-stubs==3.15.0
-    # via -r requirements/ci.txt
+    # via
+    #   -r requirements/ci.txt
+    #   -r requirements/quality.txt
 dnspython==2.6.1
     # via
     #   -r requirements/ci.txt
@@ -139,10 +136,6 @@ idna==3.7
     #   -r requirements/ci.txt
     #   -r requirements/quality.txt
     #   requests
-importlib-metadata==8.2.0
-    # via
-    #   -r requirements/pip-tools.txt
-    #   build
 iniconfig==2.0.0
     # via
     #   -r requirements/ci.txt
@@ -357,11 +350,13 @@ tox==4.17.1
 types-pyyaml==6.0.12.20240808
     # via
     #   -r requirements/ci.txt
+    #   -r requirements/quality.txt
     #   django-stubs
     #   djangorestframework-stubs
 types-requests==2.32.0.20240712
     # via
     #   -r requirements/ci.txt
+    #   -r requirements/quality.txt
     #   djangorestframework-stubs
 typing-extensions==4.12.2
     # via
@@ -373,7 +368,6 @@ typing-extensions==4.12.2
     #   django-stubs-ext
     #   djangorestframework-stubs
     #   mypy
-    #   pylint
 urllib3==2.2.2
     # via
     #   -r requirements/ci.txt
@@ -389,10 +383,6 @@ wheel==0.44.0
     # via
     #   -r requirements/pip-tools.txt
     #   pip-tools
-zipp==3.20.0
-    # via
-    #   -r requirements/pip-tools.txt
-    #   importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip