chore: improve the use of coverage (#2233)

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
         uses: actions/setup-python@v5
         with:
           cache: 'pip'
-          python-version: "3.11"
+          python-version: '3.11'
 
       - name: Install dependencies
         run: |
@@ -34,7 +34,7 @@ jobs:
     strategy:
       matrix:
         os: [ ubuntu-latest, windows-latest, macos-latest ]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
     defaults:
       run:
         shell: bash
@@ -54,13 +54,35 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt --pre
           pip install -v -e .[test] --no-build-isolation
-          if [[ ${{ matrix.os }} == windows-* ]] && [ "${{ matrix.python-version }}" == "3.10" ]; then
-            # to coverage winversioninfo
-            pip install pywin32
-          fi
 
       - name: Generate coverage report
-        run: python -m pytest --cov="cx_Freeze" --cov-report=xml
+        run: |
+          mkdir -p coverage
+          python -m pytest -nauto\
+            --cov="cx_Freeze" --cov-report=xml:./coverage/coverage0.xml
+
+      - name: Extra coverage for Windows
+        if: runner.os == 'Windows' && matrix.python-version == '3.10'
+        run: |
+          # to test lief < 0.14
+          pip install "lief==0.13.2"
+          python -m pytest -nauto\
+            --cov="cx_Freeze" --cov-report=xml:./coverage/coverage1.xml\
+            tests/test_command_build.py tests/test_command_build_exe.py\
+            tests/test_winversioninfo.py
+          # to test without lief (LIEF_DISABLED)
+          export CX_FREEZE_BIND=imagehlp
+          python -m pytest -nauto\
+            --cov="cx_Freeze" --cov-report=xml:./coverage/coverage2.xml\
+            tests/test_command_build.py tests/test_command_build_exe.py\
+            tests/test_winversioninfo.py
+          # to coverage winversioninfo using pywin32
+          pip install --upgrade pywin32
+          python -m pytest -nauto\
+            --cov="cx_Freeze" --cov-report=xml:./coverage/coverage3.xml\
+            tests/test_winversioninfo.py
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3
+        with:
+          directory: coverage

Makefile

@@ -1,27 +1,30 @@
 # Makefile to automate some tools
 
-BUILDDIR = ./build
+BUILDDIR := ./build
 EXT_SUFFIX := $(shell python -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
 PY_VERSION_NODOT := $(shell python -c "import sysconfig; print(sysconfig.get_config_var('py_version_nodot'))")
 ARCH := $(shell python -c "import platform; print(platform.machine().lower())")
+PY_PLATFORM := $(shell python -c "import sysconfig; print(sysconfig.get_platform())")
+COVERAGE_FILE := $(BUILDDIR)/.coverage-$(PY_VERSION_NODOT)-$(PY_PLATFORM)
+PRE_COMMIT_OPTIONS := --show-diff-on-failure --color=always --all-files --hook-stage=manual
 
 .PHONY: all
 all: install
 
 .PHONY: pre-commit
 pre-commit: install
-	@SKIP=pylint pre-commit run --show-diff-on-failure --color=always --all-files --hook-stage manual || true
+	@SKIP=pylint pre-commit run $(PRE_COMMIT_OPTIONS) || true
 	@pre-commit gc
 
 .PHONY: pre-commit-all
 pre-commit-all: install
-	@pre-commit run --show-diff-on-failure --color=always --all-files --hook-stage manual || true
+	@pre-commit run $(PRE_COMMIT_OPTIONS) || true
 	@pre-commit gc
 
 .PHONY: pylint
 pylint:
-	pip install --upgrade pylint
-	@pre-commit run pylint --show-diff-on-failure --color=always --all-files --hook-stage manual
+	@if ! which pylint; then pip install --upgrade pylint; fi
+	@pre-commit run pylint $(PRE_COMMIT_OPTIONS)
 
 .PHONY: clean
 clean:
@@ -31,12 +34,14 @@ clean:
 		rm -f .git/hooks/pre-commit;\
 	fi
 	@make -C doc clean
+	@COVERAGE_FILE=$(COVERAGE_FILE) coverage erase
+	@coverage erase
 
 .PHONY: install
 install:
 	if ! which pre-commit || ! [ -f .git/hooks/pre-commit ]; then\
 		python -m pip install --upgrade pip &&\
-		pip install -e .[dev,doc] &&\
+		pip install -e .[dev,doc] --no-build-isolation &&\
 		pre-commit install --install-hooks --overwrite -t pre-commit;\
 	fi
 
@@ -47,15 +52,14 @@ upgrade: clean
 	pip install --upgrade pre-commit
 	pre-commit autoupdate
 	make pre-commit
-	git diff || true
 
 .PHONY: html
 html: install
 	@if which pre-commit && [ -f .git/hooks/pre-commit ]; then\
-		pre-commit run blacken-docs -a --hook-stage manual;\
-		pre-commit run build-docs -a -v --hook-stage manual;\
+		pre-commit run blacken-docs $(PRE_COMMIT_OPTIONS);\
+		pre-commit run build-docs $(PRE_COMMIT_OPTIONS);\
 	else\
-		pip install -e .[doc] &&\
+		pip install -e .[doc] --no-build-isolation &&\
 		make -C doc html;\
 	fi
 
@@ -70,18 +74,48 @@ doc: html
 
 .PHONY: install_test
 install_test:
-	pip install -e .[test]
+	if ! which pytest || ! which cxfreeze; then\
+		pip install -e .[test] --no-build-isolation;\
+	fi
 
 .PHONY: test
 test: install_test
-	python -m pytest
+	python -m pytest -nauto --no-cov
 
 .PHONY: cov
 cov: install_test
-	python -m pytest \
-		--cov="cx_Freeze" \
-		--cov-report=html:$(BUILDDIR)/coverage \
-		--cov-report=xml:$(BUILDDIR)/coverage.xml
+	python -m pytest -nauto --cov="cx_Freeze" --cov-report=html
+	python -m webbrowser -t $(BUILDDIR)/coverage/index.html
+
+.PHONY: cov2
+cov2: install_test
+	coverage erase
+	COVERAGE_FILE=$(COVERAGE_FILE) coverage erase
+	COVERAGE_FILE=$(COVERAGE_FILE) python -m pytest -nauto --cov=cx_Freeze
+ifeq ($(PY_PLATFORM),win-amd64)
+	# Extra coverage for Windows
+	# to test lief < 0.14
+	pip install "lief==0.13.2"
+	COVERAGE_FILE=$(COVERAGE_FILE)-1 python -m pytest -nauto --cov=cx_Freeze\
+		tests/test_command_build.py tests/test_command_build_exe.py\
+		tests/test_winversioninfo.py
+	# to test without lief (LIEF_DISABLED)
+	CX_FREEZE_BIND=imagehlp \
+	COVERAGE_FILE=$(COVERAGE_FILE)-2 python -m pytest -nauto --cov=cx_Freeze\
+		--cov=cx_Freeze --cov-report=xml:./coverage/coverage2.xml\
+		tests/test_command_build.py tests/test_command_build_exe.py\
+		tests/test_winversioninfo.py
+	# to coverage winversioninfo using pywin32
+	pip install --upgrade pywin32
+	COVERAGE_FILE=$(COVERAGE_FILE)-3 python -m pytest -nauto --cov=cx_Freeze\
+		--cov=cx_Freeze --cov-report=xml:./coverage/coverage3.xml\
+		tests/test_winversioninfo.py
+	pip uninstall -y pywin32
+	pip install "lief>0.13.2"
+endif
+	coverage combine --keep $(BUILDDIR)/.coverage-*
+	rm -rf $(BUILDDIR)/coverage
+	coverage html
 	python -m webbrowser -t $(BUILDDIR)/coverage/index.html
 
 .PHONY: install_test_pre
@@ -99,10 +133,8 @@ test-pre: install_test_pre
 
 .PHONY: cov-pre
 cov-pre: install_test_pre
-	python -m pytest -o pythonpath=cx_Freeze/bases/lib-dynload/ \
-		--cov="cx_Freeze" \
-		--cov-report=html:$(BUILDDIR)/coverage \
-		--cov-report=xml:$(BUILDDIR)/coverage.xml
+	python -m pytest --cov=cx_Freeze --cov-report=html\
+		-o pythonpath=cx_Freeze/bases/lib-dynload/
 	python -m webbrowser -t $(BUILDDIR)/coverage/index.html
 
 .PHONY: release

pyproject.toml

@@ -67,9 +67,10 @@ doc = [
     "furo==2023.9.10",
 ]
 test = [
-    "pluggy==1.4.0",
     "pytest==8.0.0",
+        "pluggy==1.4.0",
     "pytest-cov==4.1.0",
+        "coverage==7.4.1",
     "pytest-datafiles==3.0.0",
     "pytest-mock==3.12.0",
     "pytest-timeout==2.2.0",
@@ -141,6 +142,27 @@ values = [
 	"prod"
 ]
 
+[tool.coverage.html]
+directory = "build/coverage"
+
+[tool.coverage.report]
+# Regexes for lines to exclude from consideration
+exclude_also = [
+    # Don't complain about missing debug-only code:
+    "def __repr__",
+
+    # Don't complain about abstract methods, they aren't run:
+    "@(abc\\.)?abstractmethod",
+]
+ignore_errors = true
+omit = [
+    "cx_Freeze/hooks/*",
+    "cx_Freeze/initscripts/*",
+]
+
+[tool.coverage.run]
+relative_files = true
+
 [tool.isort]
 profile = "black"
 line_length = 79
@@ -238,7 +260,7 @@ max-line-length = 79
 
 [tool.pytest.ini_options]
 minversion = "7.0"
-addopts = "-rpfEsXx -nauto"
+addopts = "-rpfEsXx"
 testpaths = ["tests"]
 filterwarnings = [
     "ignore::DeprecationWarning:distutils.*",

requirements-dev.txt

@@ -23,9 +23,10 @@ sphinx-new-tab-link==0.2.1
 sphinx-tabs==3.4.5
 furo==2023.9.10
 # test
-pluggy==1.4.0
 pytest==8.0.0
+pluggy==1.4.0
 pytest-cov==4.1.0
+coverage==7.4.1
 pytest-datafiles==3.0.0
 pytest-mock==3.12.0
 pytest-timeout==2.2.0

tests/README.md

@@ -5,7 +5,7 @@ directory:
 1 - Install cx_Freeze in development mode using:
 
  ```
- pip install -e .[test]
+ pip install -e .[test] --no-build-isolation
  ```
 
 1.1 - Another method:
@@ -15,32 +15,26 @@ directory:
  python setup.py develop --no-deps
  ```
 
-3 - Call the tests (debuggable) with:
+2 - Call the tests (debuggable) with:
 
  ```
- python -m pytest -n 0
+ python -m pytest --no-cov
  ```
 
-3.1 - To speed up test runs using multiple CPUs: (uses pytest-xdist)
+2.1 - To speed up test runs using multiple CPUs: (uses pytest-xdist)
 
  ```
- python -m pytest -n auto
+ python -m pytest -nauto --no-cov
  ```
 
-3.2 - With coverage (not-debuggable) with:
+3 - Call the tests with coverage (not-debuggable) with:
 
  ```
- python -m pytest --cov="cx_Freeze" --cov-report=html:./build/coverage
+ python -m pytest -nauto --cov="cx_Freeze" --cov-report=html
  ```
 
-3.2.1 - To speed up the coverage (not-debuggable) with:
+3.1 - To navigate to the coverage report:
 
  ```
- python -m pytest -n auto --cov="cx_Freeze" --cov-report=html:./build/coverage
- ```
-
-3.2.2 - To navigate to the coverage report:
-
- ```
- python -m webbrowser -t :./build/coverage/index.html
+ python -m webbrowser -t ./build/coverage/index.html
  ```