Local pypiserver default for tests (#686)

.github/workflows/create_tests_package_lists.yml

@@ -0,0 +1,31 @@
+name: Create tests package lists for offline tests
+on:
+  workflow_dispatch:
+
+jobs:
+  create_package_lists:
+    name: Create package lists
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: [3.6, 3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Upgrade pip, Install nox
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install nox
+    - name: Create lists
+      run: |
+        nox --non-interactive --session create_test_package_list-${{ matrix.python-version }} -- ./new_tests_packages
+    - name: Store reports as artifacts
+      uses: actions/upload-artifact@v2
+      with:
+        name: lists
+        path: ./new_tests_packages

.github/workflows/on-push.yml

@@ -76,6 +76,11 @@ jobs:
       with:
         path: ${{ steps.pip-cache.outputs.dir }}
         key: ${{ runner.os }}-pip${{ matrix.python-version }}
+    - name: Persistent .pipx_tests/package_cache
+      uses: actions/cache@v2
+      with:
+        path: ${{ github.workspace }}/.pipx_tests/package_cache/${{ matrix.python-version }}
+        key: pipx-tests-package-cache-${{ runner.os }}-${{ matrix.python-version }}
     - name: Upgrade pip, Install nox
       run: |
         python -m pip install --upgrade pip

.gitignore

@@ -13,3 +13,4 @@ venv
 __pycache__
 site
 docs/docs.md
+.pipx_tests

docs/changelog.md

@@ -2,6 +2,7 @@ dev
 
 - Fixed `pipx list --json` to return valid json with no venvs installed.  Previously would return and empty string to stdout. (#681)
 - Changed `pipx ensurepath` bash behavior so that only one of {`~/.profile`, `~/.bash\_profile`} is modified with the extra pipx paths, not both.  Previously, if a `.bash_profile` file was created where one didn't exist, it could cause problems, e.g. #456. The internal change is to use userpath v1.5.0 or greater. (#684)
+- Changed default nox tests, Github Workflow tests, and pytest behavior to use local pypi server with fixed lists of available packages.  This allows greater test isolation (no network pypi access needed) and determinism (fixed available dependencies.)  It also allows running the tests offline with some extra preparation beforehand (See [Running Unit Tests Offline](https://pipxproject.github.io/pipx/contributing/#running-unit-tests-offline)). The old style tests that use the internet to access pypi.org are still available using `nox -s tests_internet` or `pytest --net-pypiserver tests`. (#686)
 * Colorama is now only installed on Windows. (#691)
 
 0.16.2.1

docs/contributing.md

@@ -49,13 +49,22 @@ nox -l
 
 At the time of this writing, the output looks like this
 ```
->> nox -l
-Sessions defined in /home/csmith/git/pipx/noxfile.py:
-
-* tests-3.6
-* tests-3.7
-* tests-3.8
-* tests-3.9
+- refresh_packages_cache-3.6 -> Populate .pipx_tests/package_cache
+- refresh_packages_cache-3.7 -> Populate .pipx_tests/package_cache
+- refresh_packages_cache-3.8 -> Populate .pipx_tests/package_cache
+- refresh_packages_cache-3.9 -> Populate .pipx_tests/package_cache
+- tests_internet-3.6 -> Tests using internet pypi only
+- tests_internet-3.7 -> Tests using internet pypi only
+- tests_internet-3.8 -> Tests using internet pypi only
+- tests_internet-3.9 -> Tests using internet pypi only
+* tests-3.6 -> Tests using local pypiserver only
+* tests-3.7 -> Tests using local pypiserver only
+* tests-3.8 -> Tests using local pypiserver only
+* tests-3.9 -> Tests using local pypiserver only
+- test_all_packages-3.6
+- test_all_packages-3.7
+- test_all_packages-3.8
+- test_all_packages-3.9
 - cover -> Coverage analysis
 * lint
 - develop-3.6
@@ -69,6 +78,10 @@ Sessions defined in /home/csmith/git/pipx/noxfile.py:
 - watch_docs
 - pre_release
 - post_release
+- create_test_package_list-3.6
+- create_test_package_list-3.7
+- create_test_package_list-3.8
+- create_test_package_list-3.9
 ```
 
 ### Unit Tests
@@ -92,12 +105,61 @@ nox -s tests-3.9
 
     Coverage errors can usually be ignored when only running a subset of tests.
 
+### Running Unit Tests Offline
+
+Running the unit tests requires a directory `.pipx_tests/package_cache` to be
+populated from a fixed list of package distribution files (wheels or source
+files).  If you have network access, `nox -s tests` automatically makes sure
+this directory is populated (including downloading files if necessary) as a
+first step.  Thus if you are running the tests with network access, you can
+ignore the rest of this section.
+
+If, however, you wish to run tests offline without the need for network access,
+you can populate `.pipx_tests/package_cache` yourself manually beforehand when
+you do have network access.
+
+#### Populating the cache directory using nox
+To populate `.pipx_tests/package_cache` manually using nox, execute:
+```
+nox -s refresh_packages_cache
+```
+This will sequence through available python executable versions to populate the
+cache directory for each version of python on your platform.
+
+#### Populating the cache directory without nox
+An alternate method to populate `.pipx_tests/package_cache` without nox is to
+execute:
+```
+mkdir -p .pipx_tests/package_cache
+python3 scripts/update_package_cache.py testdata/tests_packages .pipx_tests/package_cache
+```
+You must do this using every python version that you wish to use to run the
+tests.
+
 ### Lint Tests
 
 ```
 nox -s lint
 ```
 
+### Installing or injecting new packages in tests
+If the tests are modified such that a new package / version combination is
+`pipx install`ed or `pipx inject`ed that wasn't used in other tests, then one
+must make sure it's added properly to the packages lists in
+`testdata/tests_packages`.
+
+To accomplish this:
+* Edit `testdata/tests_packages/primary_packages.txt` to add the new package(s) that you wish to `pipx install` or `pipx inject` in the tests.
+
+Then using Github workflows to generate all platforms in the Github CI:
+* Manually activate the Github workflow: Create tests package lists for offline tests
+* Download the artifact `lists` and put the files from it into `testdata/tests_packages/`
+
+Or to locally generate these lists from `testdata/tests_packages/primary_packages.txt`, on the target platform execute:
+* `nox -s create_test_package_list`
+
+Finally, check-in the new or modified list files in the directory `testdata/tests_packages`
+
 ## Testing pipx on Continuous Integration builds
 When you push a new git branch, tests will automatically be run against your code as defined in `.github/workflows/on-push.yml`.
 

noxfile.py

@@ -24,9 +24,19 @@
     "unix": [],
     "win": [],
 }
+PIPX_TESTS_CACHE_DIR = Path("./.pipx_tests/package_cache")
+PIPX_TESTS_PACKAGE_LIST_DIR = Path("testdata/tests_packages")
+
+# Platform logic
+if sys.platform == "darwin":
+    PLATFORM = "macos"
+elif sys.platform == "win32":
+    PLATFORM = "win"
+else:
+    PLATFORM = "unix"
 
 # Set nox options
-if sys.platform == "win32":
+if PLATFORM == "win":
     # build_docs fail on Windows, even if `chcp.com 65001` is used
     nox.options.sessions = ["tests", "lint"]
 else:
@@ -35,14 +45,7 @@
 
 
 def prebuild_wheels(session, prebuild_dict):
-    if sys.platform == "darwin":
-        platform = "macos"
-    elif sys.platform == "win32":
-        platform = "win"
-    else:
-        platform = "unix"
-
-    prebuild_list = prebuild_dict.get("all", []) + prebuild_dict.get(platform, [])
+    prebuild_list = prebuild_dict.get("all", []) + prebuild_dict.get(PLATFORM, [])
 
     session.install("wheel")
     wheel_dir = Path(session.virtualenv.location) / "prebuild_wheels"
@@ -85,22 +88,60 @@ def on_master_no_changes(session):
 
 
 @nox.session(python=PYTHON_ALL_VERSIONS)
-def tests(session):
+def refresh_packages_cache(session):
+    """Populate .pipx_tests/package_cache"""
+    print("Updating local tests package spec file cache...")
+    PIPX_TESTS_CACHE_DIR.mkdir(exist_ok=True, parents=True)
+    session.run(
+        "python",
+        "scripts/update_package_cache.py",
+        str(PIPX_TESTS_PACKAGE_LIST_DIR),
+        str(PIPX_TESTS_CACHE_DIR),
+    )
+
+
+def tests_with_options(session, net_pypiserver):
     session.run("python", "-m", "pip", "install", "--upgrade", "pip")
     prebuild_wheels(session, PREBUILD_PACKAGES)
     session.install("-e", ".", "pytest", "pytest-cov")
     tests = session.posargs or ["tests"]
-    session.run("pytest", "--cov=pipx", "--cov-report=", *tests)
+
+    if net_pypiserver:
+        pypiserver_option = ["--net-pypiserver"]
+    else:
+        session.install("pypiserver")
+        refresh_packages_cache(session)
+        pypiserver_option = []
+
+    session.run("pytest", *pypiserver_option, "--cov=pipx", "--cov-report=", *tests)
     session.notify("cover")
 
 
+@nox.session(python=PYTHON_ALL_VERSIONS)
+def tests_internet(session):
+    """Tests using internet pypi only"""
+    tests_with_options(session, net_pypiserver=True)
+
+
+@nox.session(python=PYTHON_ALL_VERSIONS)
+def tests(session):
+    """Tests using local pypiserver only"""
+    tests_with_options(session, net_pypiserver=False)
+
+
 @nox.session(python=PYTHON_ALL_VERSIONS)
 def test_all_packages(session):
     session.run("python", "-m", "pip", "install", "--upgrade", "pip")
     session.install("-e", ".", "pytest")
     tests = session.posargs or ["tests"]
     session.run(
-        "pytest", "-v", "--tb=no", "--show-capture=no", "--all-packages", *tests
+        "pytest",
+        "-v",
+        "--tb=no",
+        "--show-capture=no",
+        "--net-pypiserver",
+        "--all-packages",
+        *tests,
     )
 
 
@@ -214,3 +255,17 @@ def post_release(session):
         "If `git diff` above looks ok, execute the following command:\n\n"
         "    git commit -a -m 'Post-release.'\n"
     )
+
+
+@nox.session(python=PYTHON_ALL_VERSIONS)
+def create_test_package_list(session):
+    session.run("python", "-m", "pip", "install", "--upgrade", "pip")
+    output_dir = (
+        session.posargs[0] if session.posargs else str(PIPX_TESTS_PACKAGE_LIST_DIR)
+    )
+    session.run(
+        "python",
+        "scripts/list_test_packages.py",
+        str(PIPX_TESTS_PACKAGE_LIST_DIR / "primary_packages.txt"),
+        output_dir,
+    )