tests: faster testing with less duplication

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

.github/workflows/test.yml

@@ -14,6 +14,7 @@ on:
       - '*.md'
   schedule:
     - cron: "0 8 * * *"
+  workflow_dispatch:
 
 concurrency:
   group: test-${{ github.ref }}
@@ -33,6 +34,8 @@ jobs:
           - windows
         py:
           - 'pypy-3.7'
+          - 'pypy-3.8'
+          - 'pypy-3.9'
           - '3.11-dev'
           - '3.10'
           - '3.9'
@@ -41,9 +44,6 @@ jobs:
           - '3.6'
         tox-target:
           - 'tox'
-          - 'path'
-          - 'sdist'
-          - 'wheel'
           - 'min'
         exclude:
           - { py: '3.11-dev', os: macos }
@@ -64,7 +64,7 @@ jobs:
           import sys
 
           if platform.python_implementation() == "PyPy":
-              base = f"pypy{sys.version_info.major}"
+              base = f"pypy{sys.version_info.major}{sys.version_info.minor}"
           else:
               base = f"py{sys.version_info.major}{sys.version_info.minor}"
           env = f"BASE={base}\n"
@@ -87,12 +87,6 @@ jobs:
           tox -vv --notest -e ${{env.BASE}}
           tox -e ${{env.BASE}} --skip-pkg-install
 
-      - name: Run test suite via ${{ matrix.tox-target }}
-        if: matrix.tox-target != 'tox' && matrix.tox-target != 'min'
-        run: |
-          tox -vv --notest -e ${{env.BASE}}-${{ matrix.tox-target }}
-          tox -e ${{env.BASE}}-${{ matrix.tox-target }} --skip-pkg-install
-
       - name: Run minimum version test
         if: matrix.tox-target == 'min'
         run: tox -e ${{env.BASE}}-${{ matrix.tox-target }}
@@ -103,7 +97,7 @@ jobs:
         shell: bash
 
       - uses: codecov/codecov-action@v1
-        if: ${{ always() }}
+        if: always()
         env:
           PYTHON: ${{ matrix.python }}
         with:
@@ -112,6 +106,10 @@ jobs:
           env_vars: PYTHON
           name: ${{ matrix.py }} - ${{ matrix.os }}
 
+      - name: Run path test
+        if: matrix.tox-target == 'tox' && matrix.py == '3.10'
+        run: tox -e path
+
   type:
     runs-on: ubuntu-latest
     env:

pyproject.toml

@@ -25,7 +25,9 @@ build = [
 show_contexts = true
 
 [tool.pytest.ini_options]
+minversion = "6.0"
 addopts = ["--strict-config", "--strict-markers"]
+testpaths = ["tests"]
 xfail_strict = true
 junit_family = "xunit2"
 norecursedirs = "tests/integration/*"

tests/conftest.py

@@ -4,60 +4,15 @@
 import os.path
 import shutil
 import stat
-import subprocess
 import sys
 import sysconfig
 import tempfile
 
 import pytest
 
-from filelock import FileLock
-
 import build.env
 
 
-def _build_and_reinstall_build(test_mode):
-    temp = tempfile.mkdtemp()
-    try:
-        subprocess.check_output(
-            [sys.executable, '-m', 'build', f'--{test_mode}', '--no-isolation', '--outdir', temp],
-        )
-        dist_file = next(d for d in os.listdir(temp) if d.endswith('.whl' if test_mode == 'wheel' else '.tar.gz'))
-        subprocess.check_call(
-            [
-                sys.executable,
-                '-m',
-                'pip',
-                'install',
-                '--upgrade',  # ``--upgrade`` will uninstall build prior to installing the ``dist_file``
-                os.path.join(temp, dist_file),
-            ],
-        )
-    finally:
-        shutil.rmtree(temp)
-
-
-def _one_time_setup():
-    test_mode = os.environ.get('TEST_MODE')
-    if not test_mode:
-        return
-
-    if test_mode == 'path':
-        project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-        sys.path.insert(0, os.path.join(project_root, 'src'))
-    elif test_mode in {'sdist', 'wheel'}:
-        status_marker_file = os.path.join(os.environ['TEST_STATUS_DIR'], 'status-marker')
-        with FileLock(status_marker_file + '.lock'):
-            if not os.path.exists(status_marker_file):
-                _build_and_reinstall_build(test_mode)
-
-                with open(status_marker_file, 'wb'):
-                    pass
-
-
-_one_time_setup()
-
-
 def pytest_addoption(parser):
     os.environ['PYTHONWARNINGS'] = 'ignore:DEPRECATION::pip._internal.cli.base_command'  # for when not run within tox
     os.environ['PIP_DISABLE_PIP_VERSION_CHECK'] = '1'  # do not pollute stderr with upgrade advisory

tests/test_self_packaging.py

@@ -0,0 +1,103 @@
+# These tests check the sdist, path, and wheel of build to ensure that all are valid.
+
+import subprocess
+import sys
+import tarfile
+import zipfile
+
+from pathlib import Path
+
+import pytest
+
+
+DIR = Path(__file__).parent.resolve()
+MAIN_DIR = DIR.parent
+
+sdist_files = {
+    'LICENSE',
+    'PKG-INFO',
+    'README.md',
+    'pyproject.toml',
+    'setup.cfg',
+    'setup.py',
+    'src',
+    'src/build',
+    'src/build.egg-info',
+    'src/build.egg-info/PKG-INFO',
+    'src/build.egg-info/SOURCES.txt',
+    'src/build.egg-info/dependency_links.txt',
+    'src/build.egg-info/entry_points.txt',
+    'src/build.egg-info/requires.txt',
+    'src/build.egg-info/top_level.txt',
+    'src/build/__init__.py',
+    'src/build/__main__.py',
+    'src/build/env.py',
+    'src/build/py.typed',
+    'src/build/util.py',
+}
+
+wheel_files = {
+    'build/__init__.py',
+    'build/__main__.py',
+    'build/env.py',
+    'build/py.typed',
+    'build/util.py',
+    'dist-info/LICENSE',
+    'dist-info/METADATA',
+    'dist-info/RECORD',
+    'dist-info/WHEEL',
+    'dist-info/entry_points.txt',
+    'dist-info/top_level.txt',
+}
+
+
+def test_build_sdist(monkeypatch, tmpdir):
+
+    monkeypatch.chdir(MAIN_DIR)
+
+    subprocess.run(
+        [
+            sys.executable,
+            '-m',
+            'build',
+            '--sdist',
+            '--outdir',
+            str(tmpdir),
+        ],
+        check=True,
+    ).stdout
+
+    (sdist,) = tmpdir.visit('*.tar.gz')
+
+    with tarfile.open(str(sdist), 'r:gz') as tar:
+        simpler = {n.split('/', 1)[-1] for n in tar.getnames()[1:]}
+
+    assert simpler == sdist_files
+
+
+@pytest.mark.parametrize('args', ((), ('--wheel',)), ids=('from_sdist', 'direct'))
+def test_build_wheel(monkeypatch, tmpdir, args):
+
+    monkeypatch.chdir(MAIN_DIR)
+
+    subprocess.run(
+        [
+            sys.executable,
+            '-m',
+            'build',
+            *args,
+            '--outdir',
+            str(tmpdir),
+        ],
+        check=True,
+    )
+
+    (wheel,) = tmpdir.visit('*.whl')
+
+    with zipfile.ZipFile(str(wheel)) as z:
+        names = z.namelist()
+
+    trimmed = {n for n in names if 'dist-info' not in n}
+    trimmed |= {f"dist-info/{n.split('/', 1)[-1]}" for n in names if 'dist-info' in n}
+
+    assert trimmed == wheel_files

tox.ini

@@ -3,7 +3,8 @@ envlist =
     fix
     type
     docs
-    {py311, py310, py39, py38, py37, py36, pypy3}{, -path, -sdist, -wheel, -min}
+    path
+    {py311, py310, py39, py38, py37, py36, pypy37, pypy38, pypy39}{, -min}
 isolated_build = true
 skip_missing_interpreters = true
 minversion = 3.14
@@ -13,9 +14,6 @@ requires =
 [testenv]
 description =
     run test suite with {basepython}
-    path: via PYTHONPATH
-    sdist: via source distribution
-    wheel: via wheel
 passenv =
     LC_ALL
     PIP_*
@@ -24,16 +22,12 @@ setenv =
     COVERAGE_FILE = {toxworkdir}/.coverage.{envname}
     TEST_STATUS_DIR = {envtmpdir}
     PYPY3323BUG = 1
-    path: TEST_MODE = path
-    sdist: TEST_MODE = sdist
-    wheel: TEST_MODE = wheel
 extras =
     test
 commands =
     pytest -ra --cov --cov-config pyproject.toml \
       --cov-report=html:{envdir}/htmlcov --cov-context=test \
-      --cov-report=xml:{toxworkdir}/coverage.{envname}.xml \
-      tests {posargs:-n auto}
+      --cov-report=xml:{toxworkdir}/coverage.{envname}.xml {posargs:-n auto}
 
 [testenv:fix]
 description = run static analysis and style checks
@@ -48,18 +42,26 @@ commands =
     pre-commit run --all-files --show-diff-on-failure
     python -c 'print("hint: run {envdir}/bin/pre-commit install to add checks as pre-commit hook")'
 
+[testenv:path]
+description = verify build can run from source (bootstrap)
+setenv =
+    PYTHONPATH = {toxinidir}/src
+commands =
+    python -E -m pip uninstall -y build
+    pytest -ra {posargs:-n auto}
+
 [testenv:type]
 description = run type check on code base
 extras = typing
 commands =
     mypy
 
-[testenv:{py311, py310, py39, py38, py37, py36, pypy3}-min]
+[testenv:{py311, py310, py39, py38, py37, py36, pypy37, pypy38, pypy39}-min]
 description = check minimum versions required of all dependencies
 skip_install = true
 commands =
     pip install .[test] -c tests/constraints.txt
-    pytest -ra tests {posargs:-n auto}
+    pytest -ra {posargs:-n auto}
 
 [testenv:docs]
 description = build documentations
@@ -75,7 +77,6 @@ description = generate a DEV environment
 usedevelop = true
 deps =
     virtualenv>=20.0.34
-
 extras =
     doc
     test
@@ -100,7 +101,7 @@ commands =
     coverage xml -o {toxworkdir}/coverage.xml -i
     coverage html -d {toxworkdir}/htmlcov -i
     python -m diff_cover.diff_cover_tool --compare-branch {env:DIFF_AGAINST:origin/main} {toxworkdir}/coverage.xml
-depends = {py311, py310, py39, py38, py37, py36, pypy3}{, -path, -sdist, -wheel}
+depends = {py311, py310, py39, py38, py37, py36, pypy37, pypy38, pypy39}
 
 [flake8]
 max-line-length = 127