[AutoInstall] Allow wheels by default

https://bugs.webkit.org/show_bug.cgi?id=263119

Reviewed by Jonathan Bedard.

This allows packages to be installed by wheels by default; if wheel=True
is passed then only wheels will be accepted, if wheel=False is passed
then wheels will never be accepted.

* Tools/Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py:
(Package.__init__):
(Package.archives):
(AutoInstall):
(AutoInstall.enabled):
(AutoInstall.temporarily_disable):
* Tools/Scripts/webkitpy/autoinstalled/buildbot.py:
* Tools/Scripts/webkitpy/autoinstalled/twisted.py:

Canonical link: https://commits.webkit.org/269817@main

Tools/Scripts/libraries/webkitcorepy/webkitcorepy/autoinstall.py

@@ -36,6 +36,7 @@
 import zipfile
 
 from collections import defaultdict
+from contextlib import contextmanager
 from logging import NullHandler
 from webkitcorepy import log
 from webkitcorepy.version import Version
@@ -149,7 +150,7 @@ def unpack(self, target):
             else:
                 raise OSError('{} has an unrecognized package format'.format(self.path))
 
-    def __init__(self, import_name, version=None, pypi_name=None, slow_install=False, wheel=False, aliases=None, implicit_deps=None):
+    def __init__(self, import_name, version=None, pypi_name=None, slow_install=False, wheel=None, aliases=None, implicit_deps=None):
         self.name = import_name
         self.version = version
         self._archives = []
@@ -198,18 +199,34 @@ def archives(self):
                 cached_tags = None
 
                 for package in reversed(packages):
-                    if self.wheel:
+                    if self.wheel or self.wheel is None and package['name'].endswith('.whl'):
                         match = re.search(r'.+-([^-]+-[^-]+-[^-]+).whl', package['name'])
                         if not match:
                             continue
 
-                        from packaging import tags
-
-                        if not cached_tags:
-                            cached_tags = set(AutoInstall.tags())
-
-                        if all([tag not in cached_tags for tag in tags.parse_tag(match.group(1))]):
-                            continue
+                        # Temporarily disable AutoInstall so we don't try to AutoInstall
+                        # packaging via the below import while installing packaging.
+                        with AutoInstall.temporarily_disable():
+                            try:
+                                from packaging import tags
+                            except ImportError:
+                                # This is a subset of compatible tags, but these are the
+                                # only ones that are particularly common; we need these
+                                # to be able to install packaging and its dependencies.
+                                generic_tags = ["py2.py3-none-any", "py3.py2-none-any"]
+                                if sys.version_info >= (3,):
+                                    generic_tags.append("py3-none-any")
+                                else:
+                                    generic_tags.append("py2-none-any")
+
+                                if match.group(1) not in generic_tags:
+                                    continue
+                            else:
+                                if not cached_tags:
+                                    cached_tags = set(AutoInstall.tags())
+
+                                if all([tag not in cached_tags for tag in tags.parse_tag(match.group(1))]):
+                                    continue
 
                         extension = 'whl'
 
@@ -465,6 +482,7 @@ class AutoInstall(object):
     _previous_index = None
     _previous_ca_cert_path = None
     _fatal_check = False
+    _temporary_disable = 0
 
     # When sharing an install location, projects may wish to overwrite packages on disk
     # originating from a different index.
@@ -480,10 +498,19 @@ def _request(cls, url, ca_cert_path=None):
 
     @classmethod
     def enabled(cls):
+        if cls._temporary_disable > 0:
+            return False
         if os.environ.get(cls.DISABLE_ENV_VAR) not in ['0', 'FALSE', 'False', 'false', 'NO', 'No', 'no', None]:
             return False
         return True if cls.directory else None
 
+    @classmethod
+    @contextmanager
+    def temporarily_disable(cls):
+        cls._temporary_disable += 1
+        yield
+        cls._temporary_disable -= 1
+
     @classmethod
     def userspace_should_own(cls, path):
         # Windows doesn't have sudo

Tools/Scripts/webkitpy/autoinstalled/buildbot.py

@@ -33,36 +33,42 @@
 AutoInstall.install(Package('jinja2', Version(2, 11, 3), pypi_name='Jinja2'))
 AutoInstall.install(Package('pbr', Version(5, 9, 0)))
 AutoInstall.install(Package('lz4', Version(4, 3, 2)))
-AutoInstall.install(Package('PyJWT', Version(1, 7, 1), pypi_name='PyJWT', aliases=['jwt']))
+AutoInstall.install(Package('jwt', Version(1, 7, 1), pypi_name='PyJWT'))
 AutoInstall.install(Package('pyyaml', Version(5, 3, 1), pypi_name='PyYAML'))
 
 AutoInstall.install('markupsafe')
 
 if sys.version_info >= (3, 0):
-    AutoInstall.install(Package('autobahn', Version(20, 7, 1)))
+    # autobahn has wheel=False because of https://bugs.webkit.org/show_bug.cgi?id=263392
+    AutoInstall.install(Package('autobahn', Version(20, 7, 1), wheel=False))
     AutoInstall.install(Package('automat', Version(20, 2, 0), pypi_name='Automat'))
     AutoInstall.install(Package('decorator', Version(5, 1, 1)))
-    AutoInstall.install(Package('PyHamcrest', Version(2, 0, 3)))
+    AutoInstall.install(Package('hamcrest', Version(2, 0, 3), pypi_name='PyHamcrest'))
     AutoInstall.install(Package('sqlalchemy', Version(1, 3, 20), pypi_name='SQLAlchemy'))
     AutoInstall.install(Package('sqlalchemy-migrate', Version(0, 13, 0)))
     AutoInstall.install(Package('sqlparse', Version(0, 4, 2)))
     AutoInstall.install(Package('txaio', Version(20, 4, 1)))
-    AutoInstall.install(Package('Tempita', Version(0, 5, 2)))
+    AutoInstall.install(Package('tempita', Version(0, 5, 2), pypi_name='Tempita'))
 
-    AutoInstall.install(Package('buildbot', Version(2, 10, 5)))
+    # buildbot has wheel=False because we rely on items in buildbot.test that only
+    # became public API and started being included in wheels from 3.5.0.
+    AutoInstall.install(Package('buildbot', Version(2, 10, 5), wheel=False))
     AutoInstall.install(Package('buildbot-worker', Version(2, 10, 5)))
 
     from buildbot import statistics
 else:
-    AutoInstall.install(Package('autobahn', Version(17, 8, 1)))
+    # autobahn has wheel=False because of https://bugs.webkit.org/show_bug.cgi?id=263392
+    AutoInstall.install(Package('autobahn', Version(17, 8, 1), wheel=False))
     AutoInstall.install(Package('automat', Version(0, 6, 0), pypi_name='Automat'))
     AutoInstall.install(Package('decorator', Version(3, 4, 0)))
     AutoInstall.install(Package('sqlalchemy-migrate', Version(0, 11, 0)))
     AutoInstall.install(Package('sqlparse', Version(0, 2, 3)))
     AutoInstall.install(Package('txaio', Version(2, 8, 1)))
-    AutoInstall.install(Package('Tempita', Version(0, 4, 0)))
+    AutoInstall.install(Package('tempita', Version(0, 4, 0), pypi_name='Tempita'))
 
-    AutoInstall.install(Package('buildbot', Version(1, 1, 1)))
+    # buildbot has wheel=False because we rely on items in buildbot.test that only
+    # became public API and started being included in wheels from 3.5.0.
+    AutoInstall.install(Package('buildbot', Version(1, 1, 1), wheel=False))
     AutoInstall.install(Package('buildbot-worker', Version(1, 1, 1)))
 
 sys.modules[__name__] = __import__('buildbot')

Tools/Scripts/webkitpy/autoinstalled/twisted.py

@@ -33,10 +33,10 @@
 
     if sys.version_info >= (3, 11):
         AutoInstall.install(Package('twisted', Version(23, 8, 0), pypi_name='Twisted', implicit_deps=['pyparsing']))
-        AutoInstall.install(Package('pyOpenSSL', Version(23, 2, 0)))
+        AutoInstall.install(Package('OpenSSL', Version(23, 2, 0), pypi_name='pyOpenSSL'))
     else:
         AutoInstall.install(Package('twisted', Version(20, 3, 0), pypi_name='Twisted', implicit_deps=['pyparsing']))
-        AutoInstall.install(Package('pyOpenSSL', Version(20, 0, 0)))
+        AutoInstall.install(Package('OpenSSL', Version(20, 0, 0), pypi_name='pyOpenSSL'))
 
     # There are no prebuilt binaries for arm-32 of 'bcrypt' and building it requires cargo/rust
     # Since this dep is not really needed for the current arm-32 bots we skip it instead of
@@ -52,7 +52,7 @@
     AutoInstall.install(Package('incremental', Version(17, 5, 0), pypi_name='incremental'))
     AutoInstall.install(Package('twisted', Version(17, 5, 0), pypi_name='Twisted', implicit_deps=['pyparsing']))
 
-    AutoInstall.install(Package('pyOpenSSL', Version(17, 2, 0)))
+    AutoInstall.install(Package('OpenSSL', Version(17, 2, 0), pypi_name='pyOpenSSL'))
     AutoInstall.install(Package('pycparser', Version(2, 18)))
 
 sys.modules[__name__] = __import__('twisted')