feature: store version update source in embed JSON file (#2273)

docs/changelog/2265.bugfix.rst

@@ -0,0 +1,3 @@
+Try using previous updates of ``pip``, ``setuptools`` & ``wheel``
+when inside an update grace period rather than always falling back
+to embedded wheels - by :user:`mayeut`.

docs/changelog/2266.bugfix.rst

@@ -0,0 +1,2 @@
+New patch versions of ``pip``, ``setuptools`` & ``wheel`` are now
+returned in the expected timeframe. - by :user:`mayeut`.

docs/changelog/2267.bugfix.rst

@@ -0,0 +1,2 @@
+Manual upgrades of ``pip``, ``setuptools`` & ``wheel`` are
+not discarded by a periodic update - by :user:`mayeut`.

src/virtualenv/app_data/via_disk_folder.py

@@ -14,7 +14,7 @@
 │       │           └── <install class> -> CopyPipInstall / SymlinkPipInstall
 │       │               └── <wheel name> -> pip-20.1.1-py2.py3-none-any
 │       └── embed
-│           └── 1
+│           └── 2 -> json format versioning
 │               └── *.json -> for every distribution contains data about newer embed versions and releases
 └─── unzip <in zip app we cannot refer to some internal files, so first extract them>
      └── <virtualenv version>
@@ -101,7 +101,7 @@ def py_info_clear(self):
                             filename.unlink()
 
     def embed_update_log(self, distribution, for_py_version):
-        return EmbedDistributionUpdateStoreDisk(self.lock / "wheel" / for_py_version / "embed" / "1", distribution)
+        return EmbedDistributionUpdateStoreDisk(self.lock / "wheel" / for_py_version / "embed" / "2", distribution)
 
     @property
     def house(self):

src/virtualenv/seed/wheels/periodic_update.py

@@ -36,6 +36,12 @@
         pass  # pragma: no cov
 
 
+GRACE_PERIOD_CI = timedelta(hours=1)  # prevent version switch in the middle of a CI run
+GRACE_PERIOD_MINOR = timedelta(days=28)
+UPDATE_PERIOD = timedelta(days=14)
+UPDATE_ABORTED_DELAY = timedelta(hours=1)
+
+
 def periodic_update(distribution, of_version, for_py_version, wheel, search_dirs, app_data, do_periodic_update, env):
     if do_periodic_update:
         handle_auto_update(distribution, for_py_version, wheel, search_dirs, app_data, env)
@@ -48,20 +54,20 @@ def _update_wheel(ver):
         return updated_wheel
 
     u_log = UpdateLog.from_app_data(app_data, distribution, for_py_version)
-    u_log_older_than_hour = now - u_log.completed > timedelta(hours=1) if u_log.completed is not None else False
     if of_version is None:
         for _, group in groupby(u_log.versions, key=lambda v: v.wheel.version_tuple[0:2]):
-            version = next(group)  # use only latest patch version per minor, earlier assumed to be buggy
-            if wheel is not None and Path(version.filename).name == wheel.name:
-                break
-            if u_log.periodic is False or (u_log_older_than_hour and version.use(now)):
-                wheel = _update_wheel(version)
-                break
-    elif u_log.periodic is False or u_log_older_than_hour:
+            # use only latest patch version per minor, earlier assumed to be buggy
+            all_patches = list(group)
+            ignore_grace_period_minor = any(version for version in all_patches if version.use(now))
+            for version in all_patches:
+                if wheel is not None and Path(version.filename).name == wheel.name:
+                    return wheel
+                if version.use(now, ignore_grace_period_minor):
+                    return _update_wheel(version)
+    else:
         for version in u_log.versions:
             if version.wheel.version == of_version:
-                wheel = _update_wheel(version)
-                break
+                return _update_wheel(version)
 
     return wheel
 
@@ -88,41 +94,52 @@ def load_datetime(value):
 
 
 class NewVersion(object):
-    def __init__(self, filename, found_date, release_date):
+    def __init__(self, filename, found_date, release_date, source):
         self.filename = filename
         self.found_date = found_date
         self.release_date = release_date
+        self.source = source
 
     @classmethod
     def from_dict(cls, dictionary):
         return cls(
             filename=dictionary["filename"],
             found_date=load_datetime(dictionary["found_date"]),
             release_date=load_datetime(dictionary["release_date"]),
+            source=dictionary["source"],
         )
 
     def to_dict(self):
         return {
             "filename": self.filename,
             "release_date": dump_datetime(self.release_date),
             "found_date": dump_datetime(self.found_date),
+            "source": self.source,
         }
 
-    def use(self, now):
-        compare_from = self.release_date or self.found_date
-        return now - compare_from >= timedelta(days=28)
+    def use(self, now, ignore_grace_period_minor=False, ignore_grace_period_ci=False):
+        if self.source == "manual":
+            return True
+        elif self.source == "periodic":
+            if self.found_date < now - GRACE_PERIOD_CI or ignore_grace_period_ci:
+                if not ignore_grace_period_minor:
+                    compare_from = self.release_date or self.found_date
+                    return now - compare_from >= GRACE_PERIOD_MINOR
+                return True
+        return False
 
     def __repr__(self):
-        return "{}(filename={}), found_date={}, release_date={})".format(
+        return "{}(filename={}), found_date={}, release_date={}, source={})".format(
             self.__class__.__name__,
             self.filename,
             self.found_date,
             self.release_date,
+            self.source,
         )
 
     def __eq__(self, other):
         return type(self) == type(other) and all(
-            getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date"]
+            getattr(self, k) == getattr(other, k) for k in ["filename", "release_date", "found_date", "source"]
         )
 
     def __ne__(self, other):
@@ -170,12 +187,12 @@ def needs_update(self):
         if self.completed is None:  # never completed
             return self._check_start(now)
         else:
-            if now - self.completed <= timedelta(days=14):
+            if now - self.completed <= UPDATE_PERIOD:
                 return False
             return self._check_start(now)
 
     def _check_start(self, now):
-        return self.started is None or now - self.started > timedelta(hours=1)
+        return self.started is None or now - self.started > UPDATE_ABORTED_DELAY
 
 
 def trigger_update(distribution, for_py_version, wheel, search_dirs, app_data, env, periodic):
@@ -231,12 +248,24 @@ def _run_do_update(app_data, distribution, embed_filename, for_py_version, perio
     embed_update_log = app_data.embed_update_log(distribution, for_py_version)
     u_log = UpdateLog.from_dict(embed_update_log.read())
     now = datetime.now()
+    if periodic:
+        source = "periodic"
+        # mark everything not updated manually as source "periodic"
+        for version in u_log.versions:
+            if version.source != "manual":
+                version.source = source
+    else:
+        source = "manual"
+        # mark everything as source "manual"
+        for version in u_log.versions:
+            version.source = source
+
     if wheel_filename is not None:
         dest = wheelhouse / wheel_filename.name
         if not dest.exists():
             copy2(str(wheel_filename), str(wheelhouse))
     last, last_version, versions = None, None, []
-    while last is None or not last.use(now):
+    while last is None or not last.use(now, ignore_grace_period_ci=True):
         download_time = datetime.now()
         dest = acquire.download_wheel(
             distribution=distribution,
@@ -250,7 +279,7 @@ def _run_do_update(app_data, distribution, embed_filename, for_py_version, perio
         if dest is None or (u_log.versions and u_log.versions[0].filename == dest.name):
             break
         release_date = release_date_for_wheel_path(dest.path)
-        last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time)
+        last = NewVersion(filename=dest.path.name, release_date=release_date, found_date=download_time, source=source)
         logging.info("detected %s in %s", last, datetime.now() - download_time)
         versions.append(last)
         last_wheel = Wheel(Path(last.filename))

tests/unit/seed/wheels/test_bundle.py

@@ -1,12 +1,14 @@
 from __future__ import absolute_import, unicode_literals
 
 import os
+from datetime import datetime
 
 import pytest
 
 from virtualenv.app_data import AppDataDiskFolder
 from virtualenv.seed.wheels.bundle import from_bundle
 from virtualenv.seed.wheels.embed import get_embed_wheel
+from virtualenv.seed.wheels.periodic_update import dump_datetime
 from virtualenv.seed.wheels.util import Version, Wheel
 from virtualenv.util.path import Path
 
@@ -23,17 +25,19 @@ def next_pip_wheel(for_py_version):
 @pytest.fixture(scope="module")
 def app_data(tmp_path_factory, for_py_version, next_pip_wheel):
     temp_folder = tmp_path_factory.mktemp("module-app-data")
+    now = dump_datetime(datetime.now())
     app_data_ = AppDataDiskFolder(str(temp_folder))
     app_data_.embed_update_log("pip", for_py_version).write(
         {
-            "completed": "2000-01-01T00:00:00.000000Z",
+            "completed": now,
             "periodic": True,
-            "started": "2000-01-01T00:00:00.000000Z",
+            "started": now,
             "versions": [
                 {
                     "filename": next_pip_wheel.name,
                     "found_date": "2000-01-01T00:00:00.000000Z",
                     "release_date": "2000-01-01T00:00:00.000000Z",
+                    "source": "periodic",
                 }
             ],
         }