From 7e1b1934c7e0dcd400ff17a701601d912aa603bc Mon Sep 17 00:00:00 2001 From: jgoutin Date: Sat, 3 Aug 2019 10:41:34 +0200 Subject: [PATCH 01/49] Improve Visual C++ 14.X support Improve VC++14 support for VS 2017 and 2019. Separate VC from VS version (Miss match starting VS15). Improve docstrings args and returns information + fixe typos. Fix coding style and minor coding issues. Remove Microsoft "Windows SDK 7.0" dead link. --- changelog.d/1811.change.rst | 1 + setuptools/msvc.py | 1021 ++++++++++++++++++++++++----------- 2 files changed, 700 insertions(+), 322 deletions(-) create mode 100644 changelog.d/1811.change.rst diff --git a/changelog.d/1811.change.rst b/changelog.d/1811.change.rst new file mode 100644 index 0000000000..dc52c6dbbe --- /dev/null +++ b/changelog.d/1811.change.rst @@ -0,0 +1 @@ +Improve Visual C++ 14.X support, mainly for Visual Studio 2017 and 2019. \ No newline at end of file diff --git a/setuptools/msvc.py b/setuptools/msvc.py index b9c472f14f..ffa7053b64 100644 --- a/setuptools/msvc.py +++ b/setuptools/msvc.py @@ -11,13 +11,17 @@ Microsoft Visual C++ 10.0: Microsoft Windows SDK 7.1 (x86, x64, ia64) -Microsoft Visual C++ 14.0: +Microsoft Visual C++ 14.X: Microsoft Visual C++ Build Tools 2015 (x86, x64, arm) - Microsoft Visual Studio 2017 (x86, x64, arm, arm64) Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64) + Microsoft Visual Studio Build Tools 2019 (x86, x64, arm, arm64) + +This may also support compilers shipped with compatible Visual Studio versions. """ -import os +import json +from os import listdir, pathsep +from os.path import join, isfile, isdir, dirname import sys import platform import itertools @@ -30,12 +34,9 @@ if platform.system() == 'Windows': from setuptools.extern.six.moves import winreg - safe_env = os.environ + from os import environ else: - """ - Mock winreg and environ so the module can be imported - on this platform. - """ + # Mock winreg and environ so the module can be imported on this platform. class winreg: HKEY_USERS = None @@ -43,7 +44,7 @@ class winreg: HKEY_LOCAL_MACHINE = None HKEY_CLASSES_ROOT = None - safe_env = dict() + environ = dict() _msvc9_suppress_errors = ( # msvc9compiler isn't available on some platforms @@ -63,15 +64,13 @@ class winreg: def msvc9_find_vcvarsall(version): """ Patched "distutils.msvc9compiler.find_vcvarsall" to use the standalone - compiler build for Python (VCForPython). Fall back to original behavior - when the standalone compiler is not available. + compiler build for Python + (VCForPython / Microsoft Visual C++ Compiler for Python 2.7). - Redirect the path of "vcvarsall.bat". + Fall back to original behavior when the standalone compiler is not + available. - Known supported compilers - ------------------------- - Microsoft Visual C++ 9.0: - Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) + Redirect the path of "vcvarsall.bat". Parameters ---------- @@ -80,24 +79,25 @@ def msvc9_find_vcvarsall(version): Return ------ - vcvarsall.bat path: str + str + vcvarsall.bat path """ - VC_BASE = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f' - key = VC_BASE % ('', version) + vc_base = r'Software\%sMicrosoft\DevDiv\VCForPython\%0.1f' + key = vc_base % ('', version) try: # Per-user installs register the compiler path here productdir = Reg.get_value(key, "installdir") except KeyError: try: # All-user installs on a 64-bit system register here - key = VC_BASE % ('Wow6432Node\\', version) + key = vc_base % ('Wow6432Node\\', version) productdir = Reg.get_value(key, "installdir") except KeyError: productdir = None if productdir: - vcvarsall = os.path.os.path.join(productdir, "vcvarsall.bat") - if os.path.isfile(vcvarsall): + vcvarsall = join(productdir, "vcvarsall.bat") + if isfile(vcvarsall): return vcvarsall return get_unpatched(msvc9_find_vcvarsall)(version) @@ -106,20 +106,10 @@ def msvc9_find_vcvarsall(version): def msvc9_query_vcvarsall(ver, arch='x86', *args, **kwargs): """ Patched "distutils.msvc9compiler.query_vcvarsall" for support extra - compilers. + Microsoft Visual C++ 9.0 and 10.0 compilers. Set environment without use of "vcvarsall.bat". - Known supported compilers - ------------------------- - Microsoft Visual C++ 9.0: - Microsoft Visual C++ Compiler for Python 2.7 (x86, amd64) - Microsoft Windows SDK 6.1 (x86, x64, ia64) - Microsoft Windows SDK 7.0 (x86, x64, ia64) - - Microsoft Visual C++ 10.0: - Microsoft Windows SDK 7.1 (x86, x64, ia64) - Parameters ---------- ver: float @@ -129,9 +119,10 @@ def msvc9_query_vcvarsall(ver, arch='x86', *args, **kwargs): Return ------ - environment: dict + dict + environment """ - # Try to get environement from vcvarsall.bat (Classical way) + # Try to get environment from vcvarsall.bat (Classical way) try: orig = get_unpatched(msvc9_query_vcvarsall) return orig(ver, arch, *args, **kwargs) @@ -153,17 +144,10 @@ def msvc9_query_vcvarsall(ver, arch='x86', *args, **kwargs): def msvc14_get_vc_env(plat_spec): """ Patched "distutils._msvccompiler._get_vc_env" for support extra - compilers. + Microsoft Visual C++ 14.X compilers. Set environment without use of "vcvarsall.bat". - Known supported compilers - ------------------------- - Microsoft Visual C++ 14.0: - Microsoft Visual C++ Build Tools 2015 (x86, x64, arm) - Microsoft Visual Studio 2017 (x86, x64, arm, arm64) - Microsoft Visual Studio Build Tools 2017 (x86, x64, arm, arm64) - Parameters ---------- plat_spec: str @@ -171,7 +155,8 @@ def msvc14_get_vc_env(plat_spec): Return ------ - environment: dict + dict + environment """ # Try to get environment from vcvarsall.bat (Classical way) try: @@ -217,9 +202,9 @@ def _augment_exception(exc, version, arch=''): if version == 9.0: if arch.lower().find('ia64') > -1: # For VC++ 9.0, if IA64 support is needed, redirect user - # to Windows SDK 7.0 - message += ' Get it with "Microsoft Windows SDK 7.0": ' - message += msdownload % 3138 + # to Windows SDK 7.0. + # Note: No download link available from Microsoft. + message += ' Get it with "Microsoft Windows SDK 7.0"' else: # For VC++ 9.0 redirect user to Vc++ for Python 2.7 : # This redirection link is maintained by Microsoft. @@ -230,8 +215,8 @@ def _augment_exception(exc, version, arch=''): message += ' Get it with "Microsoft Windows SDK 7.1": ' message += msdownload % 8279 elif version >= 14.0: - # For VC++ 14.0 Redirect user to Visual C++ Build Tools - message += (' Get it with "Microsoft Visual C++ Build Tools": ' + # For VC++ 14.X Redirect user to latest Visual C++ Build Tools + message += (' Get it with "Build Tools for Visual Studio": ' r'https://visualstudio.microsoft.com/downloads/') exc.args = (message, ) @@ -239,26 +224,50 @@ def _augment_exception(exc, version, arch=''): class PlatformInfo: """ - Current and Target Architectures informations. + Current and Target Architectures information. Parameters ---------- arch: str Target architecture. """ - current_cpu = safe_env.get('processor_architecture', '').lower() + current_cpu = environ.get('processor_architecture', '').lower() def __init__(self, arch): self.arch = arch.lower().replace('x64', 'amd64') @property def target_cpu(self): + """ + Return Target CPU architecture. + + Return + ------ + str + Target CPU + """ return self.arch[self.arch.find('_') + 1:] def target_is_x86(self): + """ + Return True if target CPU is x86 32 bits.. + + Return + ------ + bool + CPU is x86 32 bits + """ return self.target_cpu == 'x86' def current_is_x86(self): + """ + Return True if current CPU is x86 32 bits.. + + Return + ------ + bool + CPU is x86 32 bits + """ return self.current_cpu == 'x86' def current_dir(self, hidex86=False, x64=False): @@ -274,8 +283,8 @@ def current_dir(self, hidex86=False, x64=False): Return ------ - subfolder: str - '\target', or '' (see hidex86 parameter) + str + subfolder: '\target', or '' (see hidex86 parameter) """ return ( '' if (self.current_cpu == 'x86' and hidex86) else @@ -296,8 +305,8 @@ def target_dir(self, hidex86=False, x64=False): Return ------ - subfolder: str - '\current', or '' (see hidex86 parameter) + str + subfolder: '\current', or '' (see hidex86 parameter) """ return ( '' if (self.target_cpu == 'x86' and hidex86) else @@ -312,13 +321,13 @@ def cross_dir(self, forcex86=False): Parameters ---------- forcex86: bool - Use 'x86' as current architecture even if current acritecture is + Use 'x86' as current architecture even if current architecture is not x86. Return ------ - subfolder: str - '' if target architecture is current architecture, + str + subfolder: '' if target architecture is current architecture, '\current_target' if not. """ current = 'x86' if forcex86 else self.current_cpu @@ -330,7 +339,7 @@ def cross_dir(self, forcex86=False): class RegistryInfo: """ - Microsoft Visual Studio related registry informations. + Microsoft Visual Studio related registry information. Parameters ---------- @@ -349,6 +358,11 @@ def __init__(self, platform_info): def visualstudio(self): """ Microsoft Visual Studio root registry key. + + Return + ------ + str + Registry key """ return 'VisualStudio' @@ -356,27 +370,47 @@ def visualstudio(self): def sxs(self): """ Microsoft Visual Studio SxS registry key. + + Return + ------ + str + Registry key """ - return os.path.join(self.visualstudio, 'SxS') + return join(self.visualstudio, 'SxS') @property def vc(self): """ Microsoft Visual C++ VC7 registry key. + + Return + ------ + str + Registry key """ - return os.path.join(self.sxs, 'VC7') + return join(self.sxs, 'VC7') @property def vs(self): """ Microsoft Visual Studio VS7 registry key. + + Return + ------ + str + Registry key """ - return os.path.join(self.sxs, 'VS7') + return join(self.sxs, 'VS7') @property def vc_for_python(self): """ Microsoft Visual C++ for Python registry key. + + Return + ------ + str + Registry key """ return r'DevDiv\VCForPython' @@ -384,6 +418,11 @@ def vc_for_python(self): def microsoft_sdk(self): """ Microsoft SDK registry key. + + Return + ------ + str + Registry key """ return 'Microsoft SDKs' @@ -391,20 +430,35 @@ def microsoft_sdk(self): def windows_sdk(self): """ Microsoft Windows/Platform SDK registry key. + + Return + ------ + str + Registry key """ - return os.path.join(self.microsoft_sdk, 'Windows') + return join(self.microsoft_sdk, 'Windows') @property def netfx_sdk(self): """ Microsoft .NET Framework SDK registry key. + + Return + ------ + str + Registry key """ - return os.path.join(self.microsoft_sdk, 'NETFXSDK') + return join(self.microsoft_sdk, 'NETFXSDK') @property def windows_kits_roots(self): """ Microsoft Windows Kits Roots registry key. + + Return + ------ + str + Registry key """ return r'Windows Kits\Installed Roots' @@ -421,10 +475,11 @@ def microsoft(self, key, x86=False): Return ------ - str: value + str + Registry key """ node64 = '' if self.pi.current_is_x86() or x86 else 'Wow6432Node' - return os.path.join('Software', node64, 'Microsoft', key) + return join('Software', node64, 'Microsoft', key) def lookup(self, key, name): """ @@ -439,18 +494,19 @@ def lookup(self, key, name): Return ------ - str: value + str + value """ - KEY_READ = winreg.KEY_READ + key_read = winreg.KEY_READ openkey = winreg.OpenKey ms = self.microsoft for hkey in self.HKEYS: try: - bkey = openkey(hkey, ms(key), 0, KEY_READ) + bkey = openkey(hkey, ms(key), 0, key_read) except (OSError, IOError): if not self.pi.current_is_x86(): try: - bkey = openkey(hkey, ms(key, True), 0, KEY_READ) + bkey = openkey(hkey, ms(key, True), 0, key_read) except (OSError, IOError): continue else: @@ -463,7 +519,7 @@ def lookup(self, key, name): class SystemInfo: """ - Microsoft Windows and Visual Studio related system inormations. + Microsoft Windows and Visual Studio related system information. Parameters ---------- @@ -474,30 +530,52 @@ class SystemInfo: """ # Variables and properties in this class use originals CamelCase variables - # names from Microsoft source files for more easy comparaison. - WinDir = safe_env.get('WinDir', '') - ProgramFiles = safe_env.get('ProgramFiles', '') - ProgramFilesx86 = safe_env.get('ProgramFiles(x86)', ProgramFiles) + # names from Microsoft source files for more easy comparison. + WinDir = environ.get('WinDir', '') + ProgramFiles = environ.get('ProgramFiles', '') + ProgramFilesx86 = environ.get('ProgramFiles(x86)', ProgramFiles) def __init__(self, registry_info, vc_ver=None): self.ri = registry_info self.pi = self.ri.pi - self.vc_ver = vc_ver or self._find_latest_available_vc_ver() - def _find_latest_available_vc_ver(self): - try: - return self.find_available_vc_vers()[-1] - except IndexError: - err = 'No Microsoft Visual C++ version found' - raise distutils.errors.DistutilsPlatformError(err) + self.known_vs_paths = self.find_programdata_vs_vers() + + # Except for VS15+, VC version is aligned with VS version + self.vs_ver = self.vc_ver = ( + vc_ver or self._find_latest_available_vs_ver()) + + def _find_latest_available_vs_ver(self): + """ + Find the latest VC version + + Return + ------ + float + version + """ + reg_vc_vers = self.find_reg_vs_vers() + + if not (reg_vc_vers or self.known_vs_paths): + raise distutils.errors.DistutilsPlatformError( + 'No Microsoft Visual C++ version found') + + vc_vers = set(reg_vc_vers) + vc_vers.update(self.known_vs_paths) + return sorted(vc_vers)[-1] - def find_available_vc_vers(self): + def find_reg_vs_vers(self): """ - Find all available Microsoft Visual C++ versions. + Find Microsoft Visual Studio versions available in registry. + + Return + ------ + list of float + Versions """ ms = self.ri.microsoft vckeys = (self.ri.vc, self.ri.vc_for_python, self.ri.vs) - vc_vers = [] + vs_vers = [] for hkey in self.ri.HKEYS: for key in vckeys: try: @@ -508,49 +586,108 @@ def find_available_vc_vers(self): for i in range(values): try: ver = float(winreg.EnumValue(bkey, i)[0]) - if ver not in vc_vers: - vc_vers.append(ver) + if ver not in vs_vers: + vs_vers.append(ver) except ValueError: pass for i in range(subkeys): try: ver = float(winreg.EnumKey(bkey, i)) - if ver not in vc_vers: - vc_vers.append(ver) + if ver not in vs_vers: + vs_vers.append(ver) except ValueError: pass - return sorted(vc_vers) + return sorted(vs_vers) + + def find_programdata_vs_vers(self): + r""" + Find Visual studio 2017+ versions from information in + "C:\ProgramData\Microsoft\VisualStudio\Packages\_Instances". + + Return + ------ + dict + float version as key, path as value. + """ + vs_versions = {} + instances_dir = \ + r'C:\ProgramData\Microsoft\VisualStudio\Packages\_Instances' + + try: + hashed_names = listdir(instances_dir) + + except (OSError, IOError): + # Directory not exists with all Visual Studio versions + return vs_versions + + for name in hashed_names: + try: + # Get VS installation path from "state.json" file + state_path = join(instances_dir, name, 'state.json') + with open(state_path, 'rt', encoding='utf-8') as state_file: + state = json.load(state_file) + vs_path = state['installationPath'] + + # Raises OSError if this VS installation does not contain VC + listdir(join(vs_path, r'VC\Tools\MSVC')) + + # Store version and path + vs_versions[self._as_float_version( + state['installationVersion'])] = vs_path + + except (OSError, IOError, KeyError): + # Skip if "state.json" file is missing or bad format + continue + + return vs_versions + + @staticmethod + def _as_float_version(version): + """ + Return a string version as a simplified float version (major.minor) + + Parameters + ---------- + version: str + Version. + + Return + ------ + float + version + """ + return float('.'.join(version.split('.')[:2])) @property def VSInstallDir(self): """ Microsoft Visual Studio directory. + + Return + ------ + str + path """ # Default path - name = 'Microsoft Visual Studio %0.1f' % self.vc_ver - default = os.path.join(self.ProgramFilesx86, name) + default = join(self.ProgramFilesx86, + 'Microsoft Visual Studio %0.1f' % self.vs_ver) # Try to get path from registry, if fail use default path - return self.ri.lookup(self.ri.vs, '%0.1f' % self.vc_ver) or default + return self.ri.lookup(self.ri.vs, '%0.1f' % self.vs_ver) or default @property def VCInstallDir(self): """ Microsoft Visual C++ directory. - """ - self.VSInstallDir - - guess_vc = self._guess_vc() or self._guess_vc_legacy() - - # Try to get "VC++ for Python" path from registry as default path - reg_path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) - python_vc = self.ri.lookup(reg_path, 'installdir') - default_vc = os.path.join(python_vc, 'VC') if python_vc else guess_vc - # Try to get path from registry, if fail use default path - path = self.ri.lookup(self.ri.vc, '%0.1f' % self.vc_ver) or default_vc + Return + ------ + str + path + """ + path = self._guess_vc() or self._guess_vc_legacy() - if not os.path.isdir(path): + if not isdir(path): msg = 'Microsoft Visual C++ directory not found' raise distutils.errors.DistutilsPlatformError(msg) @@ -558,186 +695,256 @@ def VCInstallDir(self): def _guess_vc(self): """ - Locate Visual C for 2017 + Locate Visual C++ for VS2017+. + + Return + ------ + str + path """ - if self.vc_ver <= 14.0: - return + if self.vs_ver <= 14.0: + return '' + + try: + # First search in known VS paths + vs_dir = self.known_vs_paths[self.vs_ver] + except KeyError: + # Else, search with path from registry + vs_dir = self.VSInstallDir + + guess_vc = join(vs_dir, r'VC\Tools\MSVC') - default = r'VC\Tools\MSVC' - guess_vc = os.path.join(self.VSInstallDir, default) # Subdir with VC exact version as name try: - vc_exact_ver = os.listdir(guess_vc)[-1] - return os.path.join(guess_vc, vc_exact_ver) + # Update the VC version with real one instead of VS version + vc_ver = listdir(guess_vc)[-1] + self.vc_ver = self._as_float_version(vc_ver) + return join(guess_vc, vc_ver) except (OSError, IOError, IndexError): - pass + return '' def _guess_vc_legacy(self): """ - Locate Visual C for versions prior to 2017 + Locate Visual C++ for versions prior to 2017. + + Return + ------ + str + path """ - default = r'Microsoft Visual Studio %0.1f\VC' % self.vc_ver - return os.path.join(self.ProgramFilesx86, default) + default = join(self.ProgramFilesx86, + r'Microsoft Visual Studio %0.1f\VC' % self.vs_ver) + + # Try to get "VC++ for Python" path from registry as default path + reg_path = join(self.ri.vc_for_python, '%0.1f' % self.vs_ver) + python_vc = self.ri.lookup(reg_path, 'installdir') + default_vc = join(python_vc, 'VC') if python_vc else default + + # Try to get path from registry, if fail use default path + return self.ri.lookup(self.ri.vc, '%0.1f' % self.vs_ver) or default_vc @property def WindowsSdkVersion(self): """ Microsoft Windows SDK versions for specified MSVC++ version. - """ - if self.vc_ver <= 9.0: - return ('7.0', '6.1', '6.0a') - elif self.vc_ver == 10.0: - return ('7.1', '7.0a') - elif self.vc_ver == 11.0: - return ('8.0', '8.0a') - elif self.vc_ver == 12.0: - return ('8.1', '8.1a') - elif self.vc_ver >= 14.0: - return ('10.0', '8.1') + + Return + ------ + tuple of str + versions + """ + if self.vs_ver <= 9.0: + return '7.0', '6.1', '6.0a' + elif self.vs_ver == 10.0: + return '7.1', '7.0a' + elif self.vs_ver == 11.0: + return '8.0', '8.0a' + elif self.vs_ver == 12.0: + return '8.1', '8.1a' + elif self.vs_ver >= 14.0: + return '10.0', '8.1' @property def WindowsSdkLastVersion(self): """ - Microsoft Windows SDK last version + Microsoft Windows SDK last version. + + Return + ------ + str + version """ - return self._use_last_dir_name(os.path.join( - self.WindowsSdkDir, 'lib')) + return self._use_last_dir_name(join(self.WindowsSdkDir, 'lib')) @property def WindowsSdkDir(self): """ Microsoft Windows SDK directory. + + Return + ------ + str + path """ sdkdir = '' for ver in self.WindowsSdkVersion: # Try to get it from registry - loc = os.path.join(self.ri.windows_sdk, 'v%s' % ver) + loc = join(self.ri.windows_sdk, 'v%s' % ver) sdkdir = self.ri.lookup(loc, 'installationfolder') if sdkdir: break - if not sdkdir or not os.path.isdir(sdkdir): + if not sdkdir or not isdir(sdkdir): # Try to get "VC++ for Python" version from registry - path = os.path.join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) + path = join(self.ri.vc_for_python, '%0.1f' % self.vc_ver) install_base = self.ri.lookup(path, 'installdir') if install_base: - sdkdir = os.path.join(install_base, 'WinSDK') - if not sdkdir or not os.path.isdir(sdkdir): + sdkdir = join(install_base, 'WinSDK') + if not sdkdir or not isdir(sdkdir): # If fail, use default new path for ver in self.WindowsSdkVersion: intver = ver[:ver.rfind('.')] - path = r'Microsoft SDKs\Windows Kits\%s' % (intver) - d = os.path.join(self.ProgramFiles, path) - if os.path.isdir(d): + path = r'Microsoft SDKs\Windows Kits\%s' % intver + d = join(self.ProgramFiles, path) + if isdir(d): sdkdir = d - if not sdkdir or not os.path.isdir(sdkdir): + if not sdkdir or not isdir(sdkdir): # If fail, use default old path for ver in self.WindowsSdkVersion: path = r'Microsoft SDKs\Windows\v%s' % ver - d = os.path.join(self.ProgramFiles, path) - if os.path.isdir(d): + d = join(self.ProgramFiles, path) + if isdir(d): sdkdir = d if not sdkdir: # If fail, use Platform SDK - sdkdir = os.path.join(self.VCInstallDir, 'PlatformSDK') + sdkdir = join(self.VCInstallDir, 'PlatformSDK') return sdkdir @property def WindowsSDKExecutablePath(self): """ Microsoft Windows SDK executable directory. + + Return + ------ + str + path """ # Find WinSDK NetFx Tools registry dir name - if self.vc_ver <= 11.0: + if self.vs_ver <= 11.0: netfxver = 35 arch = '' else: netfxver = 40 - hidex86 = True if self.vc_ver <= 12.0 else False + hidex86 = True if self.vs_ver <= 12.0 else False arch = self.pi.current_dir(x64=True, hidex86=hidex86) fx = 'WinSDK-NetFx%dTools%s' % (netfxver, arch.replace('\\', '-')) - # liste all possibles registry paths + # list all possibles registry paths regpaths = [] - if self.vc_ver >= 14.0: + if self.vs_ver >= 14.0: for ver in self.NetFxSdkVersion: - regpaths += [os.path.join(self.ri.netfx_sdk, ver, fx)] + regpaths += [join(self.ri.netfx_sdk, ver, fx)] for ver in self.WindowsSdkVersion: - regpaths += [os.path.join(self.ri.windows_sdk, 'v%sA' % ver, fx)] + regpaths += [join(self.ri.windows_sdk, 'v%sA' % ver, fx)] # Return installation folder from the more recent path for path in regpaths: execpath = self.ri.lookup(path, 'installationfolder') if execpath: - break - return execpath + return execpath @property def FSharpInstallDir(self): """ Microsoft Visual F# directory. + + Return + ------ + str + path """ - path = r'%0.1f\Setup\F#' % self.vc_ver - path = os.path.join(self.ri.visualstudio, path) + path = join(self.ri.visualstudio, r'%0.1f\Setup\F#' % self.vs_ver) return self.ri.lookup(path, 'productdir') or '' @property def UniversalCRTSdkDir(self): """ Microsoft Universal CRT SDK directory. + + Return + ------ + str + path """ # Set Kit Roots versions for specified MSVC++ version - if self.vc_ver >= 14.0: - vers = ('10', '81') - else: - vers = () + vers = ('10', '81') if self.vs_ver >= 14.0 else () # Find path of the more recent Kit for ver in vers: sdkdir = self.ri.lookup(self.ri.windows_kits_roots, 'kitsroot%s' % ver) if sdkdir: - break - return sdkdir or '' + return sdkdir or '' @property def UniversalCRTSdkLastVersion(self): """ - Microsoft Universal C Runtime SDK last version + Microsoft Universal C Runtime SDK last version. + + Return + ------ + str + version """ - return self._use_last_dir_name(os.path.join( - self.UniversalCRTSdkDir, 'lib')) + return self._use_last_dir_name(join(self.UniversalCRTSdkDir, 'lib')) @property def NetFxSdkVersion(self): """ Microsoft .NET Framework SDK versions. + + Return + ------ + tuple of str + versions """ - # Set FxSdk versions for specified MSVC++ version - if self.vc_ver >= 14.0: - return ('4.6.1', '4.6') - else: - return () + # Set FxSdk versions for specified VS version + return (('4.7.2', '4.7.1', '4.7', + '4.6.2', '4.6.1', '4.6', + '4.5.2', '4.5.1', '4.5') + if self.vs_ver >= 14.0 else ()) @property def NetFxSdkDir(self): """ Microsoft .NET Framework SDK directory. + + Return + ------ + str + path """ + sdkdir = '' for ver in self.NetFxSdkVersion: - loc = os.path.join(self.ri.netfx_sdk, ver) + loc = join(self.ri.netfx_sdk, ver) sdkdir = self.ri.lookup(loc, 'kitsinstallationfolder') if sdkdir: break - return sdkdir or '' + return sdkdir @property def FrameworkDir32(self): """ Microsoft .NET Framework 32bit directory. + + Return + ------ + str + path """ # Default path - guess_fw = os.path.join(self.WinDir, r'Microsoft.NET\Framework') + guess_fw = join(self.WinDir, r'Microsoft.NET\Framework') # Try to get path from registry, if fail use default path return self.ri.lookup(self.ri.vc, 'frameworkdir32') or guess_fw @@ -746,9 +953,14 @@ def FrameworkDir32(self): def FrameworkDir64(self): """ Microsoft .NET Framework 64bit directory. + + Return + ------ + str + path """ # Default path - guess_fw = os.path.join(self.WinDir, r'Microsoft.NET\Framework64') + guess_fw = join(self.WinDir, r'Microsoft.NET\Framework64') # Try to get path from registry, if fail use default path return self.ri.lookup(self.ri.vc, 'frameworkdir64') or guess_fw @@ -757,6 +969,11 @@ def FrameworkDir64(self): def FrameworkVersion32(self): """ Microsoft .NET Framework 32bit versions. + + Return + ------ + tuple of str + versions """ return self._find_dot_net_versions(32) @@ -764,6 +981,11 @@ def FrameworkVersion32(self): def FrameworkVersion64(self): """ Microsoft .NET Framework 64bit versions. + + Return + ------ + tuple of str + versions """ return self._find_dot_net_versions(64) @@ -775,6 +997,11 @@ def _find_dot_net_versions(self, bits): ---------- bits: int Platform number of bits: 32 or 64. + + Return + ------ + tuple of str + versions """ # Find actual .NET version in registry reg_ver = self.ri.lookup(self.ri.vc, 'frameworkver%d' % bits) @@ -782,18 +1009,17 @@ def _find_dot_net_versions(self, bits): ver = reg_ver or self._use_last_dir_name(dot_net_dir, 'v') or '' # Set .NET versions for specified MSVC++ version - if self.vc_ver >= 12.0: - frameworkver = (ver, 'v4.0') - elif self.vc_ver >= 10.0: - frameworkver = ('v4.0.30319' if ver.lower()[:2] != 'v4' else ver, - 'v3.5') - elif self.vc_ver == 9.0: - frameworkver = ('v3.5', 'v2.0.50727') - if self.vc_ver == 8.0: - frameworkver = ('v3.0', 'v2.0.50727') - return frameworkver - - def _use_last_dir_name(self, path, prefix=''): + if self.vs_ver >= 12.0: + return ver, 'v4.0' + elif self.vs_ver >= 10.0: + return 'v4.0.30319' if ver.lower()[:2] != 'v4' else ver, 'v3.5' + elif self.vs_ver == 9.0: + return 'v3.5', 'v2.0.50727' + elif self.vs_ver == 8.0: + return 'v3.0', 'v2.0.50727' + + @staticmethod + def _use_last_dir_name(path, prefix=''): """ Return name of the last dir in path or '' if no dir found. @@ -802,12 +1028,17 @@ def _use_last_dir_name(self, path, prefix=''): path: str Use dirs in this path prefix: str - Use only dirs startings by this prefix + Use only dirs starting by this prefix + + Return + ------ + str + name """ matching_dirs = ( dir_name - for dir_name in reversed(os.listdir(path)) - if os.path.isdir(os.path.join(path, dir_name)) and + for dir_name in reversed(listdir(path)) + if isdir(join(path, dir_name)) and dir_name.startswith(prefix) ) return next(matching_dirs, None) or '' @@ -818,7 +1049,7 @@ class EnvironmentInfo: Return environment variables for specified Microsoft Visual C++ version and platform : Lib, Include, Path and libpath. - This function is compatible with Microsoft Visual C++ 9.0 to 14.0. + This function is compatible with Microsoft Visual C++ 9.0 to 14.X. Script created by analysing Microsoft environment configuration files like "vcvars[...].bat", "SetEnv.Cmd", "vcbuildtools.bat", ... @@ -835,7 +1066,7 @@ class EnvironmentInfo: """ # Variables and properties in this class use originals CamelCase variables - # names from Microsoft source files for more easy comparaison. + # names from Microsoft source files for more easy comparison. def __init__(self, arch, vc_ver=None, vc_min_ver=0): self.pi = PlatformInfo(arch) @@ -846,205 +1077,255 @@ def __init__(self, arch, vc_ver=None, vc_min_ver=0): err = 'No suitable Microsoft Visual C++ version found' raise distutils.errors.DistutilsPlatformError(err) + @property + def vs_ver(self): + """ + Microsoft Visual Studio. + + Return + ------ + float + version + """ + return self.si.vs_ver + @property def vc_ver(self): """ Microsoft Visual C++ version. + + Return + ------ + float + version """ return self.si.vc_ver @property def VSTools(self): """ - Microsoft Visual Studio Tools + Microsoft Visual Studio Tools. + + Return + ------ + list of str + paths """ paths = [r'Common7\IDE', r'Common7\Tools'] - if self.vc_ver >= 14.0: + if self.vs_ver >= 14.0: arch_subdir = self.pi.current_dir(hidex86=True, x64=True) paths += [r'Common7\IDE\CommonExtensions\Microsoft\TestWindow'] paths += [r'Team Tools\Performance Tools'] paths += [r'Team Tools\Performance Tools%s' % arch_subdir] - return [os.path.join(self.si.VSInstallDir, path) for path in paths] + return [join(self.si.VSInstallDir, path) for path in paths] @property def VCIncludes(self): """ - Microsoft Visual C++ & Microsoft Foundation Class Includes + Microsoft Visual C++ & Microsoft Foundation Class Includes. + + Return + ------ + list of str + paths """ - return [os.path.join(self.si.VCInstallDir, 'Include'), - os.path.join(self.si.VCInstallDir, r'ATLMFC\Include')] + return [join(self.si.VCInstallDir, 'Include'), + join(self.si.VCInstallDir, r'ATLMFC\Include')] @property def VCLibraries(self): """ - Microsoft Visual C++ & Microsoft Foundation Class Libraries + Microsoft Visual C++ & Microsoft Foundation Class Libraries. + + Return + ------ + list of str + paths """ - if self.vc_ver >= 15.0: + if self.vs_ver >= 15.0: arch_subdir = self.pi.target_dir(x64=True) else: arch_subdir = self.pi.target_dir(hidex86=True) paths = ['Lib%s' % arch_subdir, r'ATLMFC\Lib%s' % arch_subdir] - if self.vc_ver >= 14.0: + if self.vs_ver >= 14.0: paths += [r'Lib\store%s' % arch_subdir] - return [os.path.join(self.si.VCInstallDir, path) for path in paths] + return [join(self.si.VCInstallDir, path) for path in paths] @property def VCStoreRefs(self): """ - Microsoft Visual C++ store references Libraries + Microsoft Visual C++ store references Libraries. + + Return + ------ + list of str + paths """ - if self.vc_ver < 14.0: + if self.vs_ver < 14.0: return [] - return [os.path.join(self.si.VCInstallDir, r'Lib\store\references')] + return [join(self.si.VCInstallDir, r'Lib\store\references')] @property def VCTools(self): """ - Microsoft Visual C++ Tools + Microsoft Visual C++ Tools. + + Return + ------ + list of str + paths """ si = self.si - tools = [os.path.join(si.VCInstallDir, 'VCPackages')] + tools = [join(si.VCInstallDir, 'VCPackages')] - forcex86 = True if self.vc_ver <= 10.0 else False + forcex86 = True if self.vs_ver <= 10.0 else False arch_subdir = self.pi.cross_dir(forcex86) if arch_subdir: - tools += [os.path.join(si.VCInstallDir, 'Bin%s' % arch_subdir)] + tools += [join(si.VCInstallDir, 'Bin%s' % arch_subdir)] - if self.vc_ver == 14.0: + if self.vs_ver == 14.0: path = 'Bin%s' % self.pi.current_dir(hidex86=True) - tools += [os.path.join(si.VCInstallDir, path)] + tools += [join(si.VCInstallDir, path)] - elif self.vc_ver >= 15.0: + elif self.vs_ver >= 15.0: host_dir = (r'bin\HostX86%s' if self.pi.current_is_x86() else r'bin\HostX64%s') - tools += [os.path.join( + tools += [join( si.VCInstallDir, host_dir % self.pi.target_dir(x64=True))] if self.pi.current_cpu != self.pi.target_cpu: - tools += [os.path.join( + tools += [join( si.VCInstallDir, host_dir % self.pi.current_dir(x64=True))] else: - tools += [os.path.join(si.VCInstallDir, 'Bin')] + tools += [join(si.VCInstallDir, 'Bin')] return tools @property def OSLibraries(self): """ - Microsoft Windows SDK Libraries + Microsoft Windows SDK Libraries. + + Return + ------ + list of str + paths """ - if self.vc_ver <= 10.0: + if self.vs_ver <= 10.0: arch_subdir = self.pi.target_dir(hidex86=True, x64=True) - return [os.path.join(self.si.WindowsSdkDir, 'Lib%s' % arch_subdir)] + return [join(self.si.WindowsSdkDir, 'Lib%s' % arch_subdir)] else: arch_subdir = self.pi.target_dir(x64=True) - lib = os.path.join(self.si.WindowsSdkDir, 'lib') + lib = join(self.si.WindowsSdkDir, 'lib') libver = self._sdk_subdir - return [os.path.join(lib, '%sum%s' % (libver , arch_subdir))] + return [join(lib, '%sum%s' % (libver , arch_subdir))] @property def OSIncludes(self): """ - Microsoft Windows SDK Include + Microsoft Windows SDK Include. + + Return + ------ + list of str + paths """ - include = os.path.join(self.si.WindowsSdkDir, 'include') + include = join(self.si.WindowsSdkDir, 'include') - if self.vc_ver <= 10.0: - return [include, os.path.join(include, 'gl')] + if self.vs_ver <= 10.0: + return [include, join(include, 'gl')] else: - if self.vc_ver >= 14.0: + if self.vs_ver >= 14.0: sdkver = self._sdk_subdir else: sdkver = '' - return [os.path.join(include, '%sshared' % sdkver), - os.path.join(include, '%sum' % sdkver), - os.path.join(include, '%swinrt' % sdkver)] + return [join(include, '%sshared' % sdkver), + join(include, '%sum' % sdkver), + join(include, '%swinrt' % sdkver)] @property def OSLibpath(self): """ - Microsoft Windows SDK Libraries Paths + Microsoft Windows SDK Libraries Paths. + + Return + ------ + list of str + paths """ - ref = os.path.join(self.si.WindowsSdkDir, 'References') + ref = join(self.si.WindowsSdkDir, 'References') libpath = [] - if self.vc_ver <= 9.0: + if self.vs_ver <= 9.0: libpath += self.OSLibraries - if self.vc_ver >= 11.0: - libpath += [os.path.join(ref, r'CommonConfiguration\Neutral')] + if self.vs_ver >= 11.0: + libpath += [join(ref, r'CommonConfiguration\Neutral')] - if self.vc_ver >= 14.0: + if self.vs_ver >= 14.0: libpath += [ ref, - os.path.join(self.si.WindowsSdkDir, 'UnionMetadata'), - os.path.join( - ref, - 'Windows.Foundation.UniversalApiContract', - '1.0.0.0', - ), - os.path.join( - ref, - 'Windows.Foundation.FoundationContract', - '1.0.0.0', - ), - os.path.join( - ref, - 'Windows.Networking.Connectivity.WwanContract', - '1.0.0.0', - ), - os.path.join( - self.si.WindowsSdkDir, - 'ExtensionSDKs', - 'Microsoft.VCLibs', - '%0.1f' % self.vc_ver, - 'References', - 'CommonConfiguration', - 'neutral', - ), + join(self.si.WindowsSdkDir, 'UnionMetadata'), + join(ref, 'Windows.Foundation.UniversalApiContract', '1.0.0.0'), + join(ref, 'Windows.Foundation.FoundationContract', '1.0.0.0'), + join(ref,'Windows.Networking.Connectivity.WwanContract', + '1.0.0.0'), + join(self.si.WindowsSdkDir, 'ExtensionSDKs', 'Microsoft.VCLibs', + '%0.1f' % self.vs_ver, 'References', 'CommonConfiguration', + 'neutral'), ] return libpath @property def SdkTools(self): """ - Microsoft Windows SDK Tools + Microsoft Windows SDK Tools. + + Return + ------ + list of str + paths """ return list(self._sdk_tools()) def _sdk_tools(self): """ - Microsoft Windows SDK Tools paths generator + Microsoft Windows SDK Tools paths generator. + + Return + ------ + generator of str + paths """ - if self.vc_ver < 15.0: - bin_dir = 'Bin' if self.vc_ver <= 11.0 else r'Bin\x86' - yield os.path.join(self.si.WindowsSdkDir, bin_dir) + if self.vs_ver < 15.0: + bin_dir = 'Bin' if self.vs_ver <= 11.0 else r'Bin\x86' + yield join(self.si.WindowsSdkDir, bin_dir) if not self.pi.current_is_x86(): arch_subdir = self.pi.current_dir(x64=True) path = 'Bin%s' % arch_subdir - yield os.path.join(self.si.WindowsSdkDir, path) + yield join(self.si.WindowsSdkDir, path) - if self.vc_ver == 10.0 or self.vc_ver == 11.0: + if self.vs_ver in (10.0, 11.0): if self.pi.target_is_x86(): arch_subdir = '' else: arch_subdir = self.pi.current_dir(hidex86=True, x64=True) path = r'Bin\NETFX 4.0 Tools%s' % arch_subdir - yield os.path.join(self.si.WindowsSdkDir, path) + yield join(self.si.WindowsSdkDir, path) - elif self.vc_ver >= 15.0: - path = os.path.join(self.si.WindowsSdkDir, 'Bin') + elif self.vs_ver >= 15.0: + path = join(self.si.WindowsSdkDir, 'Bin') arch_subdir = self.pi.current_dir(x64=True) sdkver = self.si.WindowsSdkLastVersion - yield os.path.join(path, '%s%s' % (sdkver, arch_subdir)) + yield join(path, '%s%s' % (sdkver, arch_subdir)) if self.si.WindowsSDKExecutablePath: yield self.si.WindowsSDKExecutablePath @@ -1052,7 +1333,12 @@ def _sdk_tools(self): @property def _sdk_subdir(self): """ - Microsoft Windows SDK version subdir + Microsoft Windows SDK version subdir. + + Return + ------ + str + subdir """ ucrtver = self.si.WindowsSdkLastVersion return ('%s\\' % ucrtver) if ucrtver else '' @@ -1060,22 +1346,32 @@ def _sdk_subdir(self): @property def SdkSetup(self): """ - Microsoft Windows SDK Setup + Microsoft Windows SDK Setup. + + Return + ------ + list of str + paths """ - if self.vc_ver > 9.0: + if self.vs_ver > 9.0: return [] - return [os.path.join(self.si.WindowsSdkDir, 'Setup')] + return [join(self.si.WindowsSdkDir, 'Setup')] @property def FxTools(self): """ - Microsoft .NET Framework Tools + Microsoft .NET Framework Tools. + + Return + ------ + list of str + paths """ pi = self.pi si = self.si - if self.vc_ver <= 10.0: + if self.vs_ver <= 10.0: include32 = True include64 = not pi.target_is_x86() and not pi.current_is_x86() else: @@ -1084,102 +1380,142 @@ def FxTools(self): tools = [] if include32: - tools += [os.path.join(si.FrameworkDir32, ver) + tools += [join(si.FrameworkDir32, ver) for ver in si.FrameworkVersion32] if include64: - tools += [os.path.join(si.FrameworkDir64, ver) + tools += [join(si.FrameworkDir64, ver) for ver in si.FrameworkVersion64] return tools @property def NetFxSDKLibraries(self): """ - Microsoft .Net Framework SDK Libraries + Microsoft .Net Framework SDK Libraries. + + Return + ------ + list of str + paths """ - if self.vc_ver < 14.0 or not self.si.NetFxSdkDir: + if self.vs_ver < 14.0 or not self.si.NetFxSdkDir: return [] arch_subdir = self.pi.target_dir(x64=True) - return [os.path.join(self.si.NetFxSdkDir, r'lib\um%s' % arch_subdir)] + return [join(self.si.NetFxSdkDir, r'lib\um%s' % arch_subdir)] @property def NetFxSDKIncludes(self): """ - Microsoft .Net Framework SDK Includes + Microsoft .Net Framework SDK Includes. + + Return + ------ + list of str + paths """ - if self.vc_ver < 14.0 or not self.si.NetFxSdkDir: + if self.vs_ver < 14.0 or not self.si.NetFxSdkDir: return [] - return [os.path.join(self.si.NetFxSdkDir, r'include\um')] + return [join(self.si.NetFxSdkDir, r'include\um')] @property def VsTDb(self): """ - Microsoft Visual Studio Team System Database + Microsoft Visual Studio Team System Database. + + Return + ------ + list of str + paths """ - return [os.path.join(self.si.VSInstallDir, r'VSTSDB\Deploy')] + return [join(self.si.VSInstallDir, r'VSTSDB\Deploy')] @property def MSBuild(self): """ - Microsoft Build Engine + Microsoft Build Engine. + + Return + ------ + list of str + paths """ - if self.vc_ver < 12.0: + if self.vs_ver < 12.0: return [] - elif self.vc_ver < 15.0: + elif self.vs_ver < 15.0: base_path = self.si.ProgramFilesx86 arch_subdir = self.pi.current_dir(hidex86=True) else: base_path = self.si.VSInstallDir arch_subdir = '' - path = r'MSBuild\%0.1f\bin%s' % (self.vc_ver, arch_subdir) - build = [os.path.join(base_path, path)] + path = r'MSBuild\%0.1f\bin%s' % (self.vs_ver, arch_subdir) + build = [join(base_path, path)] - if self.vc_ver >= 15.0: + if self.vs_ver >= 15.0: # Add Roslyn C# & Visual Basic Compiler - build += [os.path.join(base_path, path, 'Roslyn')] + build += [join(base_path, path, 'Roslyn')] return build @property def HTMLHelpWorkshop(self): """ - Microsoft HTML Help Workshop + Microsoft HTML Help Workshop. + + Return + ------ + list of str + paths """ - if self.vc_ver < 11.0: + if self.vs_ver < 11.0: return [] - return [os.path.join(self.si.ProgramFilesx86, 'HTML Help Workshop')] + return [join(self.si.ProgramFilesx86, 'HTML Help Workshop')] @property def UCRTLibraries(self): """ - Microsoft Universal C Runtime SDK Libraries + Microsoft Universal C Runtime SDK Libraries. + + Return + ------ + list of str + paths """ - if self.vc_ver < 14.0: + if self.vs_ver < 14.0: return [] arch_subdir = self.pi.target_dir(x64=True) - lib = os.path.join(self.si.UniversalCRTSdkDir, 'lib') + lib = join(self.si.UniversalCRTSdkDir, 'lib') ucrtver = self._ucrt_subdir - return [os.path.join(lib, '%sucrt%s' % (ucrtver, arch_subdir))] + return [join(lib, '%sucrt%s' % (ucrtver, arch_subdir))] @property def UCRTIncludes(self): """ - Microsoft Universal C Runtime SDK Include + Microsoft Universal C Runtime SDK Include. + + Return + ------ + list of str + paths """ - if self.vc_ver < 14.0: + if self.vs_ver < 14.0: return [] - include = os.path.join(self.si.UniversalCRTSdkDir, 'include') - return [os.path.join(include, '%sucrt' % self._ucrt_subdir)] + include = join(self.si.UniversalCRTSdkDir, 'include') + return [join(include, '%sucrt' % self._ucrt_subdir)] @property def _ucrt_subdir(self): """ - Microsoft Universal C Runtime SDK version subdir + Microsoft Universal C Runtime SDK version subdir. + + Return + ------ + str + subdir """ ucrtver = self.si.UniversalCRTSdkLastVersion return ('%s\\' % ucrtver) if ucrtver else '' @@ -1187,31 +1523,52 @@ def _ucrt_subdir(self): @property def FSharp(self): """ - Microsoft Visual F# + Microsoft Visual F#. + + Return + ------ + list of str + paths """ - if self.vc_ver < 11.0 and self.vc_ver > 12.0: + if 11.0 > self.vs_ver > 12.0: return [] - return self.si.FSharpInstallDir + return [self.si.FSharpInstallDir] @property def VCRuntimeRedist(self): """ - Microsoft Visual C++ runtime redistribuable dll - """ - arch_subdir = self.pi.target_dir(x64=True) - if self.vc_ver < 15: - redist_path = self.si.VCInstallDir - vcruntime = 'redist%s\\Microsoft.VC%d0.CRT\\vcruntime%d0.dll' - else: - redist_path = self.si.VCInstallDir.replace('\\Tools', '\\Redist') - vcruntime = 'onecore%s\\Microsoft.VC%d0.CRT\\vcruntime%d0.dll' - - # Visual Studio 2017 is still Visual C++ 14.0 - dll_ver = 14.0 if self.vc_ver == 15 else self.vc_ver + Microsoft Visual C++ runtime redistributable dll. - vcruntime = vcruntime % (arch_subdir, self.vc_ver, dll_ver) - return os.path.join(redist_path, vcruntime) + Return + ------ + str + path + """ + vcruntime = 'vcruntime%d0.dll' % self.vc_ver + arch_subdir = self.pi.target_dir(x64=True).strip('\\') + + # Installation prefixes candidates + prefixes = [] + tools_path = self.si.VCInstallDir + redist_path = dirname(tools_path.replace(r'\Tools', r'\Redist')) + if isdir(redist_path): + # Redist version may not be exactly the same as tools + redist_path = join(redist_path, listdir(redist_path)[-1]) + prefixes += [redist_path, join(redist_path, 'onecore')] + + prefixes += [join(tools_path, 'redist')] # VS14 legacy path + + # CRT directory + crt_dirs = ('Microsoft.VC%d.CRT' % (self.vc_ver * 10), + # Sometime store in directory with VS version instead of VC + 'Microsoft.VC%d.CRT' % (int(self.vs_ver) * 10)) + + # vcruntime path + for prefix, crt_dir in itertools.product(prefixes, crt_dirs): + path = join(prefix, arch_subdir, crt_dir, vcruntime) + if isfile(path): + return path def return_env(self, exists=True): """ @@ -1221,6 +1578,11 @@ def return_env(self, exists=True): ---------- exists: bool It True, only return existing paths. + + Return + ------ + dict + environment """ env = dict( include=self._build_paths('include', @@ -1254,7 +1616,7 @@ def return_env(self, exists=True): self.FSharp], exists), ) - if self.vc_ver >= 14 and os.path.isfile(self.VCRuntimeRedist): + if self.vs_ver >= 14 and isfile(self.VCRuntimeRedist): env['py_vcruntime_redist'] = self.VCRuntimeRedist return env @@ -1265,20 +1627,35 @@ def _build_paths(self, name, spec_path_lists, exists): unique, extant, directories from those paths and from the environment variable. Raise an error if no paths are resolved. + + Parameters + ---------- + name: str + Environment variable name + spec_path_lists: list of str + Paths + exists: bool + It True, only return existing paths. + + Return + ------ + str + Pathsep-separated paths """ # flatten spec_path_lists spec_paths = itertools.chain.from_iterable(spec_path_lists) - env_paths = safe_env.get(name, '').split(os.pathsep) + env_paths = environ.get(name, '').split(pathsep) paths = itertools.chain(spec_paths, env_paths) - extant_paths = list(filter(os.path.isdir, paths)) if exists else paths + extant_paths = list(filter(isdir, paths)) if exists else paths if not extant_paths: msg = "%s environment variable is empty" % name.upper() raise distutils.errors.DistutilsPlatformError(msg) unique_paths = self._unique_everseen(extant_paths) - return os.pathsep.join(unique_paths) + return pathsep.join(unique_paths) # from Python docs - def _unique_everseen(self, iterable, key=None): + @staticmethod + def _unique_everseen(iterable, key=None): """ List unique elements, preserving order. Remember all elements ever seen. From b03652f642a8ea04644eb7d5b38223148dea5611 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Tue, 13 Aug 2019 01:10:05 +0200 Subject: [PATCH 02/49] pkg_resources: fix ``Requirement`` hash/equality implementation Take PEP 508 direct URL into account. --- changelog.d/1814.change.rst | 1 + pkg_resources/__init__.py | 1 + pkg_resources/tests/test_resources.py | 17 +++++++++++++++++ 3 files changed, 19 insertions(+) create mode 100644 changelog.d/1814.change.rst diff --git a/changelog.d/1814.change.rst b/changelog.d/1814.change.rst new file mode 100644 index 0000000000..c936699dd8 --- /dev/null +++ b/changelog.d/1814.change.rst @@ -0,0 +1 @@ +Fix ``pkg_resources.Requirement`` hash/equality implementation: take PEP 508 direct URL into account. diff --git a/pkg_resources/__init__.py b/pkg_resources/__init__.py index 1f170cfda5..e75769d720 100644 --- a/pkg_resources/__init__.py +++ b/pkg_resources/__init__.py @@ -3109,6 +3109,7 @@ def __init__(self, requirement_string): self.extras = tuple(map(safe_extra, self.extras)) self.hashCmp = ( self.key, + self.url, self.specifier, frozenset(self.extras), str(self.marker) if self.marker else None, diff --git a/pkg_resources/tests/test_resources.py b/pkg_resources/tests/test_resources.py index 86afcf7411..42c801a746 100644 --- a/pkg_resources/tests/test_resources.py +++ b/pkg_resources/tests/test_resources.py @@ -520,6 +520,11 @@ def testOrdering(self): assert r1 == r2 assert str(r1) == str(r2) assert str(r2) == "Twisted==1.2c1,>=1.2" + assert ( + Requirement("Twisted") + != + Requirement("Twisted @ https://localhost/twisted.zip") + ) def testBasicContains(self): r = Requirement("Twisted>=1.2") @@ -546,11 +551,23 @@ def testOptionsAndHashing(self): == hash(( "twisted", + None, packaging.specifiers.SpecifierSet(">=1.2"), frozenset(["foo", "bar"]), None )) ) + assert ( + hash(Requirement.parse("Twisted @ https://localhost/twisted.zip")) + == + hash(( + "twisted", + "https://localhost/twisted.zip", + packaging.specifiers.SpecifierSet(), + frozenset(), + None + )) + ) def testVersionEquality(self): r1 = Requirement.parse("foo==0.3a2") From cd92d8a98f388b33fd7d5f24d096408408426b0c Mon Sep 17 00:00:00 2001 From: Christopher Head Date: Mon, 12 Aug 2019 21:37:34 -0700 Subject: [PATCH 03/49] Document using asterisk in package_data section --- docs/setuptools.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/setuptools.txt b/docs/setuptools.txt index 2e7fe3bd93..3509a301d0 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -2416,7 +2416,7 @@ tests_require list-semi include_package_data bool packages find:, find_namespace:, list-comma package_dir dict -package_data section +package_data section (1) exclude_package_data section namespace_packages list-comma py_modules list-comma @@ -2433,6 +2433,10 @@ data_files dict 40.6.0 **find_namespace directive** - The ``find_namespace:`` directive is supported since Python >=3.3. +Notes: +1. In the `package_data` section, a key named with a single asterisk (`*`) +refers to all packages, in lieu of the empty string used in `setup.py`. + Configuration API ================= From 43add1d3f5138e38adc4940647cc6eae94fb6123 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sat, 17 Aug 2019 19:14:48 -0700 Subject: [PATCH 04/49] Fixes for python3.10 --- changelog.d/1824.change.rst | 1 + pkg_resources/__init__.py | 2 +- pkg_resources/api_tests.txt | 2 +- pkg_resources/tests/test_resources.py | 2 +- setup.py | 2 +- setuptools/command/bdist_egg.py | 2 +- setuptools/command/easy_install.py | 6 +++--- setuptools/package_index.py | 2 +- setuptools/tests/test_bdist_egg.py | 2 +- 9 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 changelog.d/1824.change.rst diff --git a/changelog.d/1824.change.rst b/changelog.d/1824.change.rst new file mode 100644 index 0000000000..5f60903687 --- /dev/null +++ b/changelog.d/1824.change.rst @@ -0,0 +1 @@ +Fix tests when running under ``python3.10``. diff --git a/pkg_resources/__init__.py b/pkg_resources/__init__.py index 1f170cfda5..fb68813e0c 100644 --- a/pkg_resources/__init__.py +++ b/pkg_resources/__init__.py @@ -333,7 +333,7 @@ class UnknownExtra(ResolutionError): _provider_factories = {} -PY_MAJOR = sys.version[:3] +PY_MAJOR = '{}.{}'.format(*sys.version_info) EGG_DIST = 3 BINARY_DIST = 2 SOURCE_DIST = 1 diff --git a/pkg_resources/api_tests.txt b/pkg_resources/api_tests.txt index 0a75170e4f..7ae5a038d2 100644 --- a/pkg_resources/api_tests.txt +++ b/pkg_resources/api_tests.txt @@ -36,7 +36,7 @@ Distributions have various introspectable attributes:: >>> dist.version '0.9' - >>> dist.py_version == sys.version[:3] + >>> dist.py_version == '{}.{}'.format(*sys.version_info) True >>> print(dist.platform) diff --git a/pkg_resources/tests/test_resources.py b/pkg_resources/tests/test_resources.py index 86afcf7411..7063ed3db2 100644 --- a/pkg_resources/tests/test_resources.py +++ b/pkg_resources/tests/test_resources.py @@ -116,7 +116,7 @@ def testDistroBasics(self): self.checkFooPkg(d) d = Distribution("/some/path") - assert d.py_version == sys.version[:3] + assert d.py_version == '{}.{}'.format(*sys.version_info) assert d.platform is None def testDistroParse(self): diff --git a/setup.py b/setup.py index f5030dd670..d97895fcc0 100755 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ def _gen_console_scripts(): if any(os.environ.get(var) not in (None, "", "0") for var in var_names): return tmpl = "easy_install-{shortver} = setuptools.command.easy_install:main" - yield tmpl.format(shortver=sys.version[:3]) + yield tmpl.format(shortver='{}.{}'.format(*sys.version_info)) package_data = dict( diff --git a/setuptools/command/bdist_egg.py b/setuptools/command/bdist_egg.py index 9f8df917e6..98470f1715 100644 --- a/setuptools/command/bdist_egg.py +++ b/setuptools/command/bdist_egg.py @@ -284,7 +284,7 @@ def gen_header(self): "or refer to a module" % (ep,) ) - pyver = sys.version[:3] + pyver = '{}.{}'.format(*sys.version_info) pkg = ep.module_name full = '.'.join(ep.attrs) base = ep.attrs[0] diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 06c98271ba..593ed7776c 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -241,7 +241,7 @@ def _render_version(): """ Render the Setuptools version and installation details, then exit. """ - ver = sys.version[:3] + ver = '{}.{}'.format(*sys.version_info) dist = get_distribution('setuptools') tmpl = 'setuptools {dist.version} from {dist.location} (Python {ver})' print(tmpl.format(**locals())) @@ -1412,7 +1412,7 @@ def get_site_dirs(): os.path.join( prefix, "lib", - "python" + sys.version[:3], + "python{}.{}".format(*sys.version_info), "site-packages", ), os.path.join(prefix, "lib", "site-python"), @@ -1433,7 +1433,7 @@ def get_site_dirs(): home, 'Library', 'Python', - sys.version[:3], + '{}.{}'.format(*sys.version_info), 'site-packages', ) sitedirs.append(home_sp) diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 6b06f2ca28..f419d47167 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -46,7 +46,7 @@ _SOCKET_TIMEOUT = 15 _tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}" -user_agent = _tmpl.format(py_major=sys.version[:3], setuptools=setuptools) +user_agent = _tmpl.format(py_major='{}.{}'.format(*sys.version_info), setuptools=setuptools) def parse_requirement_arg(spec): diff --git a/setuptools/tests/test_bdist_egg.py b/setuptools/tests/test_bdist_egg.py index 54742aa646..fb5b90b1a3 100644 --- a/setuptools/tests/test_bdist_egg.py +++ b/setuptools/tests/test_bdist_egg.py @@ -42,7 +42,7 @@ def test_bdist_egg(self, setup_context, user_override): # let's see if we got our egg link at the right place [content] = os.listdir('dist') - assert re.match(r'foo-0.0.0-py[23].\d.egg$', content) + assert re.match(r'foo-0.0.0-py[23].\d+.egg$', content) @pytest.mark.xfail( os.environ.get('PYTHONDONTWRITEBYTECODE'), From 026f9b75544c53636fd692af386730553740a511 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Wed, 14 Aug 2019 02:43:53 +0200 Subject: [PATCH 05/49] docs: mention dependency links support was dropped in pip 19.0 --- docs/setuptools.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/setuptools.txt b/docs/setuptools.txt index 2e7fe3bd93..8cfb5f13a4 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -675,6 +675,10 @@ using ``setup.py develop``.) Dependencies that aren't in PyPI -------------------------------- +.. warning:: + Dependency links support has been dropped by pip starting with version + 19.0 (released 2019-01-22). + If your project depends on packages that don't exist on PyPI, you may still be able to depend on them, as long as they are available for download as: From d2db805b008346ba9aa34d8fb36faf2d019eed64 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Wed, 14 Aug 2019 02:44:20 +0200 Subject: [PATCH 06/49] docs: mention eggs are deprecated and not supported by pip --- docs/setuptools.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/setuptools.txt b/docs/setuptools.txt index 8cfb5f13a4..26a3044e82 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -1693,6 +1693,9 @@ file locations. ``bdist_egg`` - Create a Python Egg for the project =================================================== +.. warning:: + **eggs** are deprecated in favor of wheels, and not supported by pip. + This command generates a Python Egg (``.egg`` file) for the project. Python Eggs are the preferred binary distribution format for EasyInstall, because they are cross-platform (for "pure" packages), directly importable, and contain From 8b3a8d08da9780d16dce7d62433f35dbb4cc55ad Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Mon, 7 Oct 2019 18:58:40 +0200 Subject: [PATCH 07/49] add news entry --- changelog.d/1860.doc.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/1860.doc.rst diff --git a/changelog.d/1860.doc.rst b/changelog.d/1860.doc.rst new file mode 100644 index 0000000000..f3554643fa --- /dev/null +++ b/changelog.d/1860.doc.rst @@ -0,0 +1 @@ +Update documentation to mention the egg format is not supported by pip and dependency links support was dropped starting with pip 19.0. From 734d09c5a3f48338b6a3d7687c5f9d0ed02ab479 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 7 Oct 2019 16:36:54 -0400 Subject: [PATCH 08/49] Pin ordered-set to current version for consistency. --- setuptools/_vendor/vendored.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setuptools/_vendor/vendored.txt b/setuptools/_vendor/vendored.txt index 379aae56ed..5731b4244b 100644 --- a/setuptools/_vendor/vendored.txt +++ b/setuptools/_vendor/vendored.txt @@ -1,4 +1,4 @@ packaging==16.8 pyparsing==2.2.1 six==1.10.0 -ordered-set +ordered-set==3.1.1 From d7810a901382b827146874704f33bce896e1fb21 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sat, 10 Aug 2019 02:18:34 +0200 Subject: [PATCH 09/49] wheel: silence info trace when writing `requires.txt` --- setuptools/wheel.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/setuptools/wheel.py b/setuptools/wheel.py index e11f0a1d91..2982926ae8 100644 --- a/setuptools/wheel.py +++ b/setuptools/wheel.py @@ -1,6 +1,7 @@ """Wheels support.""" from distutils.util import get_platform +from distutils import log import email import itertools import os @@ -162,11 +163,17 @@ def raw_req(req): extras_require=extras_require, ), ) - write_requirements( - setup_dist.get_command_obj('egg_info'), - None, - os.path.join(egg_info, 'requires.txt'), - ) + # Temporarily disable info traces. + log_threshold = log._global_log.threshold + log.set_threshold(log.WARN) + try: + write_requirements( + setup_dist.get_command_obj('egg_info'), + None, + os.path.join(egg_info, 'requires.txt'), + ) + finally: + log.set_threshold(log_threshold) @staticmethod def _move_data_entries(destination_eggdir, dist_data): From 16a3ef93fc66373f6c5f4da12303dd111403fcb1 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Mon, 17 Sep 2018 23:40:12 +0200 Subject: [PATCH 10/49] wheel: fix installation of empty namespace package --- changelog.d/1861.change.rst | 1 + setuptools/tests/test_wheel.py | 28 ++++++++++++++++++++++++++++ setuptools/wheel.py | 4 +++- 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 changelog.d/1861.change.rst diff --git a/changelog.d/1861.change.rst b/changelog.d/1861.change.rst new file mode 100644 index 0000000000..5a4e0a56a9 --- /dev/null +++ b/changelog.d/1861.change.rst @@ -0,0 +1 @@ +Fix empty namespace package installation from wheel. diff --git a/setuptools/tests/test_wheel.py b/setuptools/tests/test_wheel.py index e85a4a7e8d..d50816c22a 100644 --- a/setuptools/tests/test_wheel.py +++ b/setuptools/tests/test_wheel.py @@ -450,6 +450,34 @@ def __repr__(self): }), ), + dict( + id='empty_namespace_package', + file_defs={ + 'foobar': { + '__init__.py': "__import__('pkg_resources').declare_namespace(__name__)", + }, + }, + setup_kwargs=dict( + namespace_packages=['foobar'], + packages=['foobar'], + ), + install_tree=flatten_tree({ + 'foo-1.0-py{py_version}.egg': [ + 'foo-1.0-py{py_version}-nspkg.pth', + {'EGG-INFO': [ + 'PKG-INFO', + 'RECORD', + 'WHEEL', + 'namespace_packages.txt', + 'top_level.txt', + ]}, + {'foobar': [ + '__init__.py', + ]}, + ] + }), + ), + dict( id='data_in_package', file_defs={ diff --git a/setuptools/wheel.py b/setuptools/wheel.py index 2982926ae8..22eec05ecd 100644 --- a/setuptools/wheel.py +++ b/setuptools/wheel.py @@ -213,6 +213,8 @@ def _fix_namespace_packages(egg_info, destination_eggdir): for mod in namespace_packages: mod_dir = os.path.join(destination_eggdir, *mod.split('.')) mod_init = os.path.join(mod_dir, '__init__.py') - if os.path.exists(mod_dir) and not os.path.exists(mod_init): + if not os.path.exists(mod_dir): + os.mkdir(mod_dir) + if not os.path.exists(mod_init): with open(mod_init, 'w') as fp: fp.write(NAMESPACE_PACKAGE_INIT) From cb8769d7d1a694d37194c44b98c543b2d5d38fc5 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Wed, 26 Jun 2019 22:25:03 +0200 Subject: [PATCH 11/49] minor cleanup --- setuptools/command/easy_install.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index 06c98271ba..d7b7566c0d 100644 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -1180,8 +1180,7 @@ def _set_fetcher_options(self, base): # to the setup.cfg file. ei_opts = self.distribution.get_option_dict('easy_install').copy() fetch_directives = ( - 'find_links', 'site_dirs', 'index_url', 'optimize', - 'site_dirs', 'allow_hosts', + 'find_links', 'site_dirs', 'index_url', 'optimize', 'allow_hosts', ) fetch_options = {} for key, val in ei_opts.items(): From 0d831c90e345ba6e7d0f82954f0cec5bdaa8501b Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Sat, 10 Aug 2019 03:57:58 +0200 Subject: [PATCH 12/49] improve workaround for #1644 Make it possible to use a more recent version of pip for tests. --- pytest.ini | 2 +- tests/requirements.txt | 1 - tools/tox_pip.py | 29 +++++++++++++++++++++++++++++ tox.ini | 14 ++++++++------ 4 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 tools/tox_pip.py diff --git a/pytest.ini b/pytest.ini index 612fb91f63..764a55dd1f 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ [pytest] addopts=--doctest-modules --doctest-glob=pkg_resources/api_tests.txt -r sxX -norecursedirs=dist build *.egg setuptools/extern pkg_resources/extern .* +norecursedirs=dist build *.egg setuptools/extern pkg_resources/extern tools .* flake8-ignore = setuptools/site-patch.py F821 setuptools/py*compat.py F811 diff --git a/tests/requirements.txt b/tests/requirements.txt index cb3e672696..1f70adee0a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -9,4 +9,3 @@ coverage>=4.5.1 pytest-cov>=2.5.1 paver; python_version>="3.6" futures; python_version=="2.7" -pip==18.1 # Temporary workaround for #1644. diff --git a/tools/tox_pip.py b/tools/tox_pip.py new file mode 100644 index 0000000000..1117f99653 --- /dev/null +++ b/tools/tox_pip.py @@ -0,0 +1,29 @@ +import os +import shutil +import subprocess +import sys +from glob import glob + +VIRTUAL_ENV = os.environ['VIRTUAL_ENV'] +TOX_PIP_DIR = os.path.join(VIRTUAL_ENV, 'pip') + + +def pip(args): + # First things first, get a recent (stable) version of pip. + if not os.path.exists(TOX_PIP_DIR): + subprocess.check_call([sys.executable, '-m', 'pip', + '--disable-pip-version-check', + 'install', '-t', TOX_PIP_DIR, + 'pip']) + shutil.rmtree(glob(os.path.join(TOX_PIP_DIR, 'pip-*.dist-info'))[0]) + # And use that version. + for n, a in enumerate(args): + if not a.startswith('-'): + if a in 'install' and '-e' in args[n:]: + args.insert(n + 1, '--no-use-pep517') + break + subprocess.check_call([sys.executable, os.path.join(TOX_PIP_DIR, 'pip')] + args) + + +if __name__ == '__main__': + pip(sys.argv[1:]) diff --git a/tox.ini b/tox.ini index e0eef95a45..8b34c235c9 100644 --- a/tox.ini +++ b/tox.ini @@ -7,14 +7,16 @@ [tox] envlist=python +[helpers] +# Wrapper for calls to pip that make sure the version being used is a +# up-to-date, and to prevent the current working directory from being +# added to `sys.path`. +pip = python {toxinidir}/tools/tox_pip.py + [testenv] deps=-rtests/requirements.txt -# Changed from default (`python -m pip ...`) -# to prevent the current working directory -# from being added to `sys.path`. -install_command=python -c 'import sys; sys.path.remove(""); from pkg_resources import load_entry_point; load_entry_point("pip", "console_scripts", "pip")()' install {opts} {packages} -# Same as above. -list_dependencies_command={envbindir}/pip freeze --all +install_command = {[helpers]pip} install {opts} {packages} +list_dependencies_command = {[helpers]pip} freeze --all setenv=COVERAGE_FILE={toxworkdir}/.coverage.{envname} # TODO: The passed environment variables came from copying other tox.ini files # These should probably be individually annotated to explain what needs them. From b5a9209cd2a3882204e0f9c9158a4a4d1ebf5e9b Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Wed, 14 Aug 2019 02:01:37 +0200 Subject: [PATCH 13/49] docs: drop ez_setup documentation and related references --- changelog.d/1862.doc.rst | 1 + docs/easy_install.txt | 4 +- docs/ez_setup.txt | 195 --------------------------------------- docs/setuptools.txt | 29 +----- 4 files changed, 4 insertions(+), 225 deletions(-) create mode 100644 changelog.d/1862.doc.rst delete mode 100644 docs/ez_setup.txt diff --git a/changelog.d/1862.doc.rst b/changelog.d/1862.doc.rst new file mode 100644 index 0000000000..b71583ba37 --- /dev/null +++ b/changelog.d/1862.doc.rst @@ -0,0 +1 @@ +Drop ez_setup documentation: deprecated for some time (last updated in 2016), and still relying on easy_install (deprecated too). diff --git a/docs/easy_install.txt b/docs/easy_install.txt index aa11f89083..e247d8fd76 100644 --- a/docs/easy_install.txt +++ b/docs/easy_install.txt @@ -317,8 +317,8 @@ Note that instead of changing your ``PATH`` to include the Python scripts directory, you can also retarget the installation location for scripts so they go on a directory that's already on the ``PATH``. For more information see `Command-Line Options`_ and `Configuration Files`_. During installation, -pass command line options (such as ``--script-dir``) to -``ez_setup.py`` to control where ``easy_install.exe`` will be installed. +pass command line options (such as ``--script-dir``) to to control where +scripts will be installed. Windows Executable Launcher diff --git a/docs/ez_setup.txt b/docs/ez_setup.txt deleted file mode 100644 index 0126fee307..0000000000 --- a/docs/ez_setup.txt +++ /dev/null @@ -1,195 +0,0 @@ -:orphan: - -``ez_setup`` distribution guide -=============================== - -Using ``setuptools``... Without bundling it! ---------------------------------------------- - -.. warning:: **ez_setup** is deprecated in favor of PIP with **PEP-518** support. - -.. _ez_setup.py: https://bootstrap.pypa.io/ez_setup.py - -.. _EasyInstall Installation Instructions: easy_install.html - -.. _Custom Installation Locations: easy_install.html - -Your users might not have ``setuptools`` installed on their machines, or even -if they do, it might not be the right version. Fixing this is easy; just -download `ez_setup.py`_, and put it in the same directory as your ``setup.py`` -script. (Be sure to add it to your revision control system, too.) Then add -these two lines to the very top of your setup script, before the script imports -anything from setuptools: - -.. code-block:: python - - import ez_setup - ez_setup.use_setuptools() - -That's it. The ``ez_setup`` module will automatically download a matching -version of ``setuptools`` from PyPI, if it isn't present on the target system. -Whenever you install an updated version of setuptools, you should also update -your projects' ``ez_setup.py`` files, so that a matching version gets installed -on the target machine(s). - -(By the way, if you need to distribute a specific version of ``setuptools``, -you can specify the exact version and base download URL as parameters to the -``use_setuptools()`` function. See the function's docstring for details.) - - -What Your Users Should Know ---------------------------- - -In general, a setuptools-based project looks just like any distutils-based -project -- as long as your users have an internet connection and are installing -to ``site-packages``, that is. But for some users, these conditions don't -apply, and they may become frustrated if this is their first encounter with -a setuptools-based project. To keep these users happy, you should review the -following topics in your project's installation instructions, if they are -relevant to your project and your target audience isn't already familiar with -setuptools and ``easy_install``. - -Network Access - If your project is using ``ez_setup``, you should inform users of the - need to either have network access, or to preinstall the correct version of - setuptools using the `EasyInstall installation instructions`_. Those - instructions also have tips for dealing with firewalls as well as how to - manually download and install setuptools. - -Custom Installation Locations - You should inform your users that if they are installing your project to - somewhere other than the main ``site-packages`` directory, they should - first install setuptools using the instructions for `Custom Installation - Locations`_, before installing your project. - -Your Project's Dependencies - If your project depends on other projects that may need to be downloaded - from PyPI or elsewhere, you should list them in your installation - instructions, or tell users how to find out what they are. While most - users will not need this information, any users who don't have unrestricted - internet access may have to find, download, and install the other projects - manually. (Note, however, that they must still install those projects - using ``easy_install``, or your project will not know they are installed, - and your setup script will try to download them again.) - - If you want to be especially friendly to users with limited network access, - you may wish to build eggs for your project and its dependencies, making - them all available for download from your site, or at least create a page - with links to all of the needed eggs. In this way, users with limited - network access can manually download all the eggs to a single directory, - then use the ``-f`` option of ``easy_install`` to specify the directory - to find eggs in. Users who have full network access can just use ``-f`` - with the URL of your download page, and ``easy_install`` will find all the - needed eggs using your links directly. This is also useful when your - target audience isn't able to compile packages (e.g. most Windows users) - and your package or some of its dependencies include C code. - -Revision Control System Users and Co-Developers - Users and co-developers who are tracking your in-development code using - a revision control system should probably read this manual's sections - regarding such development. Alternately, you may wish to create a - quick-reference guide containing the tips from this manual that apply to - your particular situation. For example, if you recommend that people use - ``setup.py develop`` when tracking your in-development code, you should let - them know that this needs to be run after every update or commit. - - Similarly, if you remove modules or data files from your project, you - should remind them to run ``setup.py clean --all`` and delete any obsolete - ``.pyc`` or ``.pyo``. (This tip applies to the distutils in general, not - just setuptools, but not everybody knows about them; be kind to your users - by spelling out your project's best practices rather than leaving them - guessing.) - -Creating System Packages - Some users want to manage all Python packages using a single package - manager, and sometimes that package manager isn't ``easy_install``! - Setuptools currently supports ``bdist_rpm``, ``bdist_wininst``, and - ``bdist_dumb`` formats for system packaging. If a user has a locally- - installed "bdist" packaging tool that internally uses the distutils - ``install`` command, it should be able to work with ``setuptools``. Some - examples of "bdist" formats that this should work with include the - ``bdist_nsi`` and ``bdist_msi`` formats for Windows. - - However, packaging tools that build binary distributions by running - ``setup.py install`` on the command line or as a subprocess will require - modification to work with setuptools. They should use the - ``--single-version-externally-managed`` option to the ``install`` command, - combined with the standard ``--root`` or ``--record`` options. - See the `install command`_ documentation below for more details. The - ``bdist_deb`` command is an example of a command that currently requires - this kind of patching to work with setuptools. - - Please note that building system packages may require you to install - some system software, for example ``bdist_rpm`` requires the ``rpmbuild`` - command to be installed. - - If you or your users have a problem building a usable system package for - your project, please report the problem via the mailing list so that - either the "bdist" tool in question or setuptools can be modified to - resolve the issue. - -Your users might not have ``setuptools`` installed on their machines, or even -if they do, it might not be the right version. Fixing this is easy; just -download `ez_setup.py`_, and put it in the same directory as your ``setup.py`` -script. (Be sure to add it to your revision control system, too.) Then add -these two lines to the very top of your setup script, before the script imports -anything from setuptools: - -.. code-block:: python - - import ez_setup - ez_setup.use_setuptools() - -That's it. The ``ez_setup`` module will automatically download a matching -version of ``setuptools`` from PyPI, if it isn't present on the target system. -Whenever you install an updated version of setuptools, you should also update -your projects' ``ez_setup.py`` files, so that a matching version gets installed -on the target machine(s). - -(By the way, if you need to distribute a specific version of ``setuptools``, -you can specify the exact version and base download URL as parameters to the -``use_setuptools()`` function. See the function's docstring for details.) - -.. _install command: - -``install`` - Run ``easy_install`` or old-style installation -============================================================ - -The setuptools ``install`` command is basically a shortcut to run the -``easy_install`` command on the current project. However, for convenience -in creating "system packages" of setuptools-based projects, you can also -use this option: - -``--single-version-externally-managed`` - This boolean option tells the ``install`` command to perform an "old style" - installation, with the addition of an ``.egg-info`` directory so that the - installed project will still have its metadata available and operate - normally. If you use this option, you *must* also specify the ``--root`` - or ``--record`` options (or both), because otherwise you will have no way - to identify and remove the installed files. - -This option is automatically in effect when ``install`` is invoked by another -distutils command, so that commands like ``bdist_wininst`` and ``bdist_rpm`` -will create system packages of eggs. It is also automatically in effect if -you specify the ``--root`` option. - - -``install_egg_info`` - Install an ``.egg-info`` directory in ``site-packages`` -============================================================================== - -Setuptools runs this command as part of ``install`` operations that use the -``--single-version-externally-managed`` options. You should not invoke it -directly; it is documented here for completeness and so that distutils -extensions such as system package builders can make use of it. This command -has only one option: - -``--install-dir=DIR, -d DIR`` - The parent directory where the ``.egg-info`` directory will be placed. - Defaults to the same as the ``--install-dir`` option specified for the - ``install_lib`` command, which is usually the system ``site-packages`` - directory. - -This command assumes that the ``egg_info`` command has been given valid options -via the command line or ``setup.cfg``, as it will invoke the ``egg_info`` -command and use its options to locate the project's source ``.egg-info`` -directory. diff --git a/docs/setuptools.txt b/docs/setuptools.txt index 26a3044e82..ba6b170a11 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -8,14 +8,7 @@ distribute Python packages, especially ones that have dependencies on other packages. Packages built and distributed using ``setuptools`` look to the user like -ordinary Python packages based on the ``distutils``. Your users don't need to -install or even know about setuptools in order to use them, and you don't -have to include the entire setuptools package in your distributions. By -including just a single `bootstrap module`_ (a 12K .py file), your package will -automatically download and install ``setuptools`` if the user is building your -package from source and doesn't have a suitable version already installed. - -.. _bootstrap module: https://bootstrap.pypa.io/ez_setup.py +ordinary Python packages based on the ``distutils``. Feature Highlights: @@ -62,8 +55,6 @@ Feature Highlights: .. contents:: **Table of Contents** -.. _ez_setup.py: `bootstrap module`_ - ----------------- Developer's Guide @@ -596,10 +587,6 @@ Python must be available via the ``PATH`` environment variable, under its "long" name. That is, if the egg is built for Python 2.3, there must be a ``python2.3`` executable present in a directory on ``PATH``. -This feature is primarily intended to support ez_setup the installation of -setuptools itself on non-Windows platforms, but may also be useful for other -projects as well. - IMPORTANT NOTE: Eggs with an "eggsecutable" header cannot be renamed, or invoked via symlinks. They *must* be invoked using their original filename, in order to ensure that, once running, ``pkg_resources`` will know what project @@ -1263,20 +1250,6 @@ To install your newly uploaded package ``example_pkg``, you can use pip:: If you have issues at any point, please refer to `Packaging project tutorials`_ for clarification. -Distributing legacy ``setuptools`` projects using ez_setup.py -------------------------------------------------------------- - -.. warning:: **ez_setup** is deprecated in favor of PIP with **PEP-518** support. - -Distributing packages using the legacy ``ez_setup.py`` and ``easy_install`` is -deprecated in favor of PIP. Please consider migrating to using pip and twine based -distribution. - -However, if you still have any ``ez_setup`` based packages, documentation for -ez_setup based distributions can be found at `ez_setup distribution guide`_. - -.. _ez_setup distribution guide: ez_setup.html - Setting the ``zip_safe`` flag ----------------------------- From 6a73945462a7e73168be53e377e4110a5dc89fb9 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Wed, 14 Aug 2019 00:49:50 +0200 Subject: [PATCH 14/49] tests: drop (easy_install based) manual tests --- tests/manual_test.py | 100 ------------------------------------------- 1 file changed, 100 deletions(-) delete mode 100644 tests/manual_test.py diff --git a/tests/manual_test.py b/tests/manual_test.py deleted file mode 100644 index 99db4b0143..0000000000 --- a/tests/manual_test.py +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env python - -import sys -import os -import shutil -import tempfile -import subprocess -from distutils.command.install import INSTALL_SCHEMES -from string import Template - -from setuptools.extern.six.moves import urllib - - -def _system_call(*args): - assert subprocess.call(args) == 0 - - -def tempdir(func): - def _tempdir(*args, **kwargs): - test_dir = tempfile.mkdtemp() - old_dir = os.getcwd() - os.chdir(test_dir) - try: - return func(*args, **kwargs) - finally: - os.chdir(old_dir) - shutil.rmtree(test_dir) - - return _tempdir - - -SIMPLE_BUILDOUT = """\ -[buildout] - -parts = eggs - -[eggs] -recipe = zc.recipe.egg - -eggs = - extensions -""" - -BOOTSTRAP = 'http://downloads.buildout.org/1/bootstrap.py' -PYVER = sys.version.split()[0][:3] - -_VARS = {'base': '.', - 'py_version_short': PYVER} - -scheme = 'nt' if sys.platform == 'win32' else 'unix_prefix' -PURELIB = INSTALL_SCHEMES[scheme]['purelib'] - - -@tempdir -def test_virtualenv(): - """virtualenv with setuptools""" - purelib = os.path.abspath(Template(PURELIB).substitute(**_VARS)) - _system_call('virtualenv', '--no-site-packages', '.') - _system_call('bin/easy_install', 'setuptools==dev') - # linux specific - site_pkg = os.listdir(purelib) - site_pkg.sort() - assert 'setuptools' in site_pkg[0] - easy_install = os.path.join(purelib, 'easy-install.pth') - with open(easy_install) as f: - res = f.read() - assert 'setuptools' in res - - -@tempdir -def test_full(): - """virtualenv + pip + buildout""" - _system_call('virtualenv', '--no-site-packages', '.') - _system_call('bin/easy_install', '-q', 'setuptools==dev') - _system_call('bin/easy_install', '-qU', 'setuptools==dev') - _system_call('bin/easy_install', '-q', 'pip') - _system_call('bin/pip', 'install', '-q', 'zc.buildout') - - with open('buildout.cfg', 'w') as f: - f.write(SIMPLE_BUILDOUT) - - with open('bootstrap.py', 'w') as f: - f.write(urllib.request.urlopen(BOOTSTRAP).read()) - - _system_call('bin/python', 'bootstrap.py') - _system_call('bin/buildout', '-q') - eggs = os.listdir('eggs') - eggs.sort() - assert len(eggs) == 3 - assert eggs[1].startswith('setuptools') - del eggs[1] - assert eggs == [ - 'extensions-0.3-py2.6.egg', - 'zc.recipe.egg-1.2.2-py2.6.egg', - ] - - -if __name__ == '__main__': - test_virtualenv() - test_full() From 4c0e204413e6d2c33662a02c73a0ad13d81af9e5 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Tue, 8 Oct 2019 19:32:09 +0200 Subject: [PATCH 15/49] doc: drop most references to EasyInstall --- changelog.d/1868.doc.rst | 1 + docs/development.txt | 2 +- docs/formats.txt | 7 +- docs/pkg_resources.txt | 14 +-- docs/setuptools.txt | 178 ++++++++++----------------------------- 5 files changed, 54 insertions(+), 148 deletions(-) create mode 100644 changelog.d/1868.doc.rst diff --git a/changelog.d/1868.doc.rst b/changelog.d/1868.doc.rst new file mode 100644 index 0000000000..82283d7aea --- /dev/null +++ b/changelog.d/1868.doc.rst @@ -0,0 +1 @@ +Drop most documentation references to (deprecated) EasyInstall. diff --git a/docs/development.txt b/docs/development.txt index 455f038afa..28e653fea9 100644 --- a/docs/development.txt +++ b/docs/development.txt @@ -7,7 +7,7 @@ Authority (PyPA) and led by Jason R. Coombs. This document describes the process by which Setuptools is developed. This document assumes the reader has some passing familiarity with -*using* setuptools, the ``pkg_resources`` module, and EasyInstall. It +*using* setuptools, the ``pkg_resources`` module, and pip. It does not attempt to explain basic concepts like inter-project dependencies, nor does it contain detailed lexical syntax for most file formats. Neither does it explain concepts like "namespace diff --git a/docs/formats.txt b/docs/formats.txt index a182eb99fc..6c0456de29 100644 --- a/docs/formats.txt +++ b/docs/formats.txt @@ -299,11 +299,8 @@ specified by the ``setup_requires`` parameter to the Distribution. A list of dependency URLs, one per line, as specified using the ``dependency_links`` keyword to ``setup()``. These may be direct download URLs, or the URLs of web pages containing direct download -links, and will be used by EasyInstall to find dependencies, as though -the user had manually provided them via the ``--find-links`` command -line option. Please see the setuptools manual and EasyInstall manual -for more information on specifying this option, and for information on -how EasyInstall processes ``--find-links`` URLs. +links. Please see the setuptools manual for more information on +specifying this option. ``depends.txt`` -- Obsolete, do not create! diff --git a/docs/pkg_resources.txt b/docs/pkg_resources.txt index 806f1b1468..b887a9239f 100644 --- a/docs/pkg_resources.txt +++ b/docs/pkg_resources.txt @@ -245,8 +245,8 @@ abbreviation for ``pkg_resources.working_set.require()``: interactive interpreter hacking than for production use. If you're creating an actual library or application, it's strongly recommended that you create a "setup.py" script using ``setuptools``, and declare all your requirements - there. That way, tools like EasyInstall can automatically detect what - requirements your package has, and deal with them accordingly. + there. That way, tools like pip can automatically detect what requirements + your package has, and deal with them accordingly. Note that calling ``require('SomePackage')`` will not install ``SomePackage`` if it isn't already present. If you need to do this, you @@ -611,9 +611,9 @@ Requirements Parsing or activation of both Report-O-Rama and any libraries it needs in order to provide PDF support. For example, you could use:: - easy_install.py Report-O-Rama[PDF] + pip install Report-O-Rama[PDF] - To install the necessary packages using the EasyInstall program, or call + To install the necessary packages using pip, or call ``pkg_resources.require('Report-O-Rama[PDF]')`` to add the necessary distributions to sys.path at runtime. @@ -1843,9 +1843,9 @@ History because it isn't necessarily a filesystem path (and hasn't been for some time now). The ``location`` of ``Distribution`` objects in the filesystem should always be normalized using ``pkg_resources.normalize_path()``; all - of the setuptools and EasyInstall code that generates distributions from - the filesystem (including ``Distribution.from_filename()``) ensure this - invariant, but if you use a more generic API like ``Distribution()`` or + of the setuptools' code that generates distributions from the filesystem + (including ``Distribution.from_filename()``) ensure this invariant, but if + you use a more generic API like ``Distribution()`` or ``Distribution.from_location()`` you should take care that you don't create a distribution with an un-normalized filesystem path. diff --git a/docs/setuptools.txt b/docs/setuptools.txt index 26a3044e82..8468c3ad03 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -19,12 +19,6 @@ package from source and doesn't have a suitable version already installed. Feature Highlights: -* Automatically find/download/install/upgrade dependencies at build time using - the `EasyInstall tool `_, - which supports downloading via HTTP, FTP, Subversion, and SourceForge, and - automatically scans web pages linked from PyPI to find download links. (It's - the closest thing to CPAN currently available for Python.) - * Create `Python Eggs `_ - a single-file importable distribution format @@ -73,10 +67,6 @@ Developer's Guide Installing ``setuptools`` ========================= -.. _EasyInstall Installation Instructions: easy_install.html - -.. _Custom Installation Locations: easy_install.html - .. _Installing Packages: https://packaging.python.org/tutorials/installing-packages/ To install the latest version of setuptools, use:: @@ -160,7 +150,7 @@ Specifying Your Project's Version Setuptools can work well with most versioning schemes; there are, however, a few special things to watch out for, in order to ensure that setuptools and -EasyInstall can always tell what version of your package is newer than another +other tools can always tell what version of your package is newer than another version. Knowing these things will also help you correctly specify what versions of other projects your project depends on. @@ -301,11 +291,10 @@ unless you need the associated ``setuptools`` feature. ``setup_requires`` A string or list of strings specifying what other distributions need to be present in order for the *setup script* to run. ``setuptools`` will - attempt to obtain these (even going so far as to download them using - ``EasyInstall``) before processing the rest of the setup script or commands. - This argument is needed if you are using distutils extensions as part of - your build process; for example, extensions that process setup() arguments - and turn them into EGG-INFO metadata files. + attempt to obtain these before processing the rest of the setup script or + commands. This argument is needed if you are using distutils extensions as + part of your build process; for example, extensions that process setup() + arguments and turn them into EGG-INFO metadata files. (Note: projects listed in ``setup_requires`` will NOT be automatically installed on the system where the setup script is being run. They are @@ -318,8 +307,7 @@ unless you need the associated ``setuptools`` feature. A list of strings naming URLs to be searched when satisfying dependencies. These links will be used if needed to install packages specified by ``setup_requires`` or ``tests_require``. They will also be written into - the egg's metadata for use by tools like EasyInstall to use when installing - an ``.egg`` file. + the egg's metadata for use during install by tools that support them. ``namespace_packages`` A list of strings naming the project's "namespace packages". A namespace @@ -351,8 +339,7 @@ unless you need the associated ``setuptools`` feature. needed to install it, you can use this option to specify them. It should be a string or list of strings specifying what other distributions need to be present for the package's tests to run. When you run the ``test`` - command, ``setuptools`` will attempt to obtain these (even going - so far as to download them using ``EasyInstall``). Note that these + command, ``setuptools`` will attempt to obtain these. Note that these required projects will *not* be installed on the system where the tests are run, but only downloaded to the project's setup directory if they're not already installed locally. @@ -552,11 +539,12 @@ script called ``baz``, you might do something like this:: ) When this project is installed on non-Windows platforms (using "setup.py -install", "setup.py develop", or by using EasyInstall), a set of ``foo``, -``bar``, and ``baz`` scripts will be installed that import ``main_func`` and -``some_func`` from the specified modules. The functions you specify are called -with no arguments, and their return value is passed to ``sys.exit()``, so you -can return an errorlevel or message to print to stderr. +install", "setup.py develop", or with pip), a set of ``foo``, ``bar``, +and ``baz`` scripts will be installed that import ``main_func`` and +``some_func`` from the specified modules. The functions you specify are +called with no arguments, and their return value is passed to +``sys.exit()``, so you can return an errorlevel or message to print to +stderr. On Windows, a set of ``foo.exe``, ``bar.exe``, and ``baz.exe`` launchers are created, alongside a set of ``foo.py``, ``bar.py``, and ``baz.pyw`` files. The @@ -613,7 +601,7 @@ Declaring Dependencies ``setuptools`` supports automatically installing dependencies when a package is installed, and including information about dependencies in Python Eggs (so that -package management tools like EasyInstall can use the information). +package management tools like pip can use the information). ``setuptools`` and ``pkg_resources`` use a common syntax for specifying a project's required dependencies. This syntax consists of a project's PyPI @@ -652,10 +640,9 @@ requirement in a string, each requirement must begin on a new line. This has three effects: -1. When your project is installed, either by using EasyInstall, ``setup.py - install``, or ``setup.py develop``, all of the dependencies not already - installed will be located (via PyPI), downloaded, built (if necessary), - and installed. +1. When your project is installed, either by using pip, ``setup.py install``, + or ``setup.py develop``, all of the dependencies not already installed will + be located (via PyPI), downloaded, built (if necessary), and installed. 2. Any scripts in your project will be installed with wrappers that verify the availability of the specified dependencies at runtime, and ensure that @@ -729,9 +716,8 @@ This will do a checkout (or a clone, in Git and Mercurial parlance) to a temporary folder and run ``setup.py bdist_egg``. The ``dependency_links`` option takes the form of a list of URL strings. For -example, the below will cause EasyInstall to search the specified page for -eggs or source distributions, if the package's dependencies aren't already -installed:: +example, this will cause a search of the specified page for eggs or source +distributions, if the package's dependencies aren't already installed:: setup( ... @@ -771,7 +757,7 @@ names of "extra" features, to strings or lists of strings describing those features' requirements. These requirements will *not* be automatically installed unless another package depends on them (directly or indirectly) by including the desired "extras" in square brackets after the associated project -name. (Or if the extras were listed in a requirement spec on the EasyInstall +name. (Or if the extras were listed in a requirement spec on the "pip install" command line.) Extras can be used by a project's `entry points`_ to specify dynamic @@ -1186,13 +1172,12 @@ preferred way of working (as opposed to using a common independent staging area or the site-packages directory). To do this, use the ``setup.py develop`` command. It works very similarly to -``setup.py install`` or the EasyInstall tool, except that it doesn't actually -install anything. Instead, it creates a special ``.egg-link`` file in the -deployment directory, that links to your project's source code. And, if your -deployment directory is Python's ``site-packages`` directory, it will also -update the ``easy-install.pth`` file to include your project's source code, -thereby making it available on ``sys.path`` for all programs using that Python -installation. +``setup.py install``, except that it doesn't actually install anything. +Instead, it creates a special ``.egg-link`` file in the deployment directory, +that links to your project's source code. And, if your deployment directory is +Python's ``site-packages`` directory, it will also update the +``easy-install.pth`` file to include your project's source code, thereby making +it available on ``sys.path`` for all programs using that Python installation. If you have enabled the ``use_2to3`` flag, then of course the ``.egg-link`` will not link directly to your source code when run under Python 3, since @@ -1312,20 +1297,14 @@ you've checked over all the warnings it issued, and you are either satisfied it doesn't work, you can always change it to ``False``, which will force ``setuptools`` to install your project as a directory rather than as a zipfile. -Of course, the end-user can still override either decision, if they are using -EasyInstall to install your package. And, if you want to override for testing -purposes, you can just run ``setup.py easy_install --zip-ok .`` or ``setup.py -easy_install --always-unzip .`` in your project directory. to install the -package as a zipfile or directory, respectively. - In the future, as we gain more experience with different packages and become more satisfied with the robustness of the ``pkg_resources`` runtime, the "zip safety" analysis may become less conservative. However, we strongly recommend that you determine for yourself whether your project functions correctly when installed as a zipfile, correct any problems if you can, and then make an explicit declaration of ``True`` or ``False`` for the ``zip_safe`` -flag, so that it will not be necessary for ``bdist_egg`` or ``EasyInstall`` to -try to guess whether your project can work as a zipfile. +flag, so that it will not be necessary for ``bdist_egg`` to try to guess +whether your project can work as a zipfile. .. _Namespace Packages: @@ -1439,9 +1418,9 @@ to generate a daily build or snapshot for. See the section below on the (Also, before you release your project, be sure to see the section above on `Specifying Your Project's Version`_ for more information about how pre- and -post-release tags affect how setuptools and EasyInstall interpret version -numbers. This is important in order to make sure that dependency processing -tools will know which versions of your project are newer than others.) +post-release tags affect how version numbers are interpreted. This is +important in order to make sure that dependency processing tools will know +which versions of your project are newer than others.) Finally, if you are creating builds frequently, and either building them in a downloadable location or are copying them to a distribution server, you should @@ -1497,58 +1476,6 @@ all practical purposes, you'll probably use only the ``--formats`` option, if you use any option at all. -Making your package available for EasyInstall ---------------------------------------------- - -There may be reasons why you don't want to upload distributions to -PyPI, and just want your existing distributions (or perhaps a Subversion -checkout) to be used instead. - -There are three ``setup()`` arguments that affect EasyInstall: - -``url`` and ``download_url`` - These become links on your project's PyPI page. EasyInstall will examine - them to see if they link to a package ("primary links"), or whether they are - HTML pages. If they're HTML pages, EasyInstall scans all HREF's on the - page for primary links - -``long_description`` - EasyInstall will check any URLs contained in this argument to see if they - are primary links. - -A URL is considered a "primary link" if it is a link to a .tar.gz, .tgz, .zip, -.egg, .egg.zip, .tar.bz2, or .exe file, or if it has an ``#egg=project`` or -``#egg=project-version`` fragment identifier attached to it. EasyInstall -attempts to determine a project name and optional version number from the text -of a primary link *without* downloading it. When it has found all the primary -links, EasyInstall will select the best match based on requested version, -platform compatibility, and other criteria. - -So, if your ``url`` or ``download_url`` point either directly to a downloadable -source distribution, or to HTML page(s) that have direct links to such, then -EasyInstall will be able to locate downloads automatically. If you want to -make Subversion checkouts available, then you should create links with either -``#egg=project`` or ``#egg=project-version`` added to the URL. You should -replace ``project`` and ``version`` with the values they would have in an egg -filename. (Be sure to actually generate an egg and then use the initial part -of the filename, rather than trying to guess what the escaped form of the -project name and version number will be.) - -Note that Subversion checkout links are of lower precedence than other kinds -of distributions, so EasyInstall will not select a Subversion checkout for -downloading unless it has a version included in the ``#egg=`` suffix, and -it's a higher version than EasyInstall has seen in any other links for your -project. - -As a result, it's a common practice to use mark checkout URLs with a version of -"dev" (i.e., ``#egg=projectname-dev``), so that users can do something like -this:: - - easy_install --editable projectname==dev - -in order to check out the in-development version of ``projectname``. - - Making "Official" (Non-Snapshot) Releases ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -1803,9 +1730,9 @@ Here are some of the options that the ``develop`` command accepts. Note that they affect the project's dependencies as well as the project itself, so if you have dependencies that need to be installed and you use ``--exclude-scripts`` (for example), the dependencies' scripts will not be installed either! For -this reason, you may want to use EasyInstall to install the project's -dependencies before using the ``develop`` command, if you need finer control -over the installation options for dependencies. +this reason, you may want to use pip to install the project's dependencies +before using the ``develop`` command, if you need finer control over the +installation options for dependencies. ``--uninstall, -u`` Un-deploy the current project. You may use the ``--install-dir`` or ``-d`` @@ -1815,10 +1742,10 @@ over the installation options for dependencies. staging area is Python's ``site-packages`` directory. Note that this option currently does *not* uninstall script wrappers! You - must uninstall them yourself, or overwrite them by using EasyInstall to - activate a different version of the package. You can also avoid installing - script wrappers in the first place, if you use the ``--exclude-scripts`` - (aka ``-x``) option when you run ``develop`` to deploy the project. + must uninstall them yourself, or overwrite them by using pip to install a + different version of the package. You can also avoid installing script + wrappers in the first place, if you use the ``--exclude-scripts`` (aka + ``-x``) option when you run ``develop`` to deploy the project. ``--multi-version, -m`` "Multi-version" mode. Specifying this option prevents ``develop`` from @@ -1827,8 +1754,8 @@ over the installation options for dependencies. removed upon successful deployment. In multi-version mode, no specific version of the package is available for importing, unless you use ``pkg_resources.require()`` to put it on ``sys.path``, or you are running - a wrapper script generated by ``setuptools`` or EasyInstall. (In which - case the wrapper script calls ``require()`` for you.) + a wrapper script generated by ``setuptools``. (In which case the wrapper + script calls ``require()`` for you.) Note that if you install to a directory other than ``site-packages``, this option is automatically in effect, because ``.pth`` files can only be @@ -1881,25 +1808,6 @@ files), the ``develop`` command will use them as defaults, unless you override them in a ``[develop]`` section or on the command line. -``easy_install`` - Find and install packages -============================================ - -This command runs the `EasyInstall tool -`_ for you. It is exactly -equivalent to running the ``easy_install`` command. All command line arguments -following this command are consumed and not processed further by the distutils, -so this must be the last command listed on the command line. Please see -the EasyInstall documentation for the options reference and usage examples. -Normally, there is no reason to use this command via the command line, as you -can just use ``easy_install`` directly. It's only listed here so that you know -it's a distutils command, which means that you can: - -* create command aliases that use it, -* create distutils extensions that invoke it as a subcommand, and -* configure options for it in your ``setup.cfg`` or other distutils config - files. - - .. _egg_info: ``egg_info`` - Create egg metadata and set build tags @@ -1958,9 +1866,9 @@ added in the following order: (Note: Because these options modify the version number used for source and binary distributions of your project, you should first make sure that you know how the resulting version numbers will be interpreted by automated tools -like EasyInstall. See the section above on `Specifying Your Project's -Version`_ for an explanation of pre- and post-release tags, as well as tips on -how to choose and verify a versioning scheme for your your project.) +like pip. See the section above on `Specifying Your Project's Version`_ for an +explanation of pre- and post-release tags, as well as tips on how to choose and +verify a versioning scheme for your your project.) For advanced uses, there is one other option that can be set, to change the location of the project's ``.egg-info`` directory. Commands that need to find From 1410d87f8abb5bb28bf97f53219ee0db7b6340a3 Mon Sep 17 00:00:00 2001 From: isidentical Date: Fri, 4 Oct 2019 19:18:54 +0300 Subject: [PATCH 16/49] Upgrade setuptools.depends to importlib from depracated imp --- setuptools/depends.py | 89 +++++++++++++++++++++++----- setuptools/tests/test_integration.py | 1 + 2 files changed, 76 insertions(+), 14 deletions(-) diff --git a/setuptools/depends.py b/setuptools/depends.py index 45e7052d8d..97f0ed9dae 100644 --- a/setuptools/depends.py +++ b/setuptools/depends.py @@ -1,11 +1,23 @@ import sys -import imp import marshal from distutils.version import StrictVersion -from imp import PKG_DIRECTORY, PY_COMPILED, PY_SOURCE, PY_FROZEN +from setuptools.extern import six from .py33compat import Bytecode +if six.PY2: + import imp + from imp import PKG_DIRECTORY, PY_COMPILED, PY_SOURCE, PY_FROZEN +else: + import os.path + from importlib.util import find_spec, spec_from_loader + from importlib.machinery import SOURCE_SUFFIXES, BYTECODE_SUFFIXES, EXTENSION_SUFFIXES, BuiltinImporter, FrozenImporter + PY_SOURCE = 1 + PY_COMPILED = 2 + C_EXTENSION = 3 + C_BUILTIN = 6 + PY_FROZEN = 7 + __all__ = [ 'Require', 'find_module', 'get_module_constant', 'extract_constant' @@ -81,21 +93,59 @@ def is_current(self, paths=None): def find_module(module, paths=None): """Just like 'imp.find_module()', but with package support""" + if six.PY3: + spec = find_spec(module, paths) + if spec is None: + raise ImportError("Can't find %s" % module) + if not spec.has_location and hasattr(spec, 'submodule_search_locations'): + spec = spec_from_loader('__init__.py', spec.loader) + + kind = -1 + file = None + static = isinstance(spec.loader, type) + if spec.origin == 'frozen' or static and issubclass(spec.loader, FrozenImporter): + kind = PY_FROZEN + path = None # imp compabilty + suffix = mode = '' # imp compability + elif spec.origin == 'built-in' or static and issubclass(spec.loader, BuiltinImporter): + kind = C_BUILTIN + path = None # imp compabilty + suffix = mode = '' # imp compability + elif spec.has_location: + frozen = False + path = spec.origin + suffix = os.path.splitext(path)[1] + mode = 'r' if suffix in SOURCE_SUFFIXES else 'rb' + + if suffix in SOURCE_SUFFIXES: + kind = PY_SOURCE + elif suffix in BYTECODE_SUFFIXES: + kind = PY_COMPILED + elif suffix in EXTENSION_SUFFIXES: + kind = C_EXTENSION + + if kind in {PY_SOURCE, PY_COMPILED}: + file = open(path, mode) + else: + path = None + suffix = mode= '' - parts = module.split('.') + return file, path, (suffix, mode, kind) - while parts: - part = parts.pop(0) - f, path, (suffix, mode, kind) = info = imp.find_module(part, paths) + else: + parts = module.split('.') + while parts: + part = parts.pop(0) + f, path, (suffix, mode, kind) = info = imp.find_module(part, paths) - if kind == PKG_DIRECTORY: - parts = parts or ['__init__'] - paths = [path] + if kind == PKG_DIRECTORY: + parts = parts or ['__init__'] + paths = [path] - elif parts: - raise ImportError("Can't find %r in %s" % (parts, module)) + elif parts: + raise ImportError("Can't find %r in %s" % (parts, module)) - return info + return info def get_module_constant(module, symbol, default=-1, paths=None): @@ -111,18 +161,29 @@ def get_module_constant(module, symbol, default=-1, paths=None): # Module doesn't exist return None + if six.PY3: + spec = find_spec(module, paths) + if hasattr(spec, 'submodule_search_locations'): + spec = spec_from_loader('__init__.py', spec.loader) + try: if kind == PY_COMPILED: f.read(8) # skip magic & date code = marshal.load(f) elif kind == PY_FROZEN: - code = imp.get_frozen_object(module) + if six.PY2: + code = imp.get_frozen_object(module) + else: + code = spec.loader.get_code(module) elif kind == PY_SOURCE: code = compile(f.read(), path, 'exec') else: # Not something we can parse; we'll have to import it. :( if module not in sys.modules: - imp.load_module(module, f, path, (suffix, mode, kind)) + if six.PY2: + imp.load_module(module, f, path, (suffix, mode, kind)) + else: + sys.modules[module] = module_from_spec(spec) return getattr(sys.modules[module], symbol, None) finally: diff --git a/setuptools/tests/test_integration.py b/setuptools/tests/test_integration.py index 1e13218899..1c0b2b18bb 100644 --- a/setuptools/tests/test_integration.py +++ b/setuptools/tests/test_integration.py @@ -143,6 +143,7 @@ def test_build_deps_on_distutils(request, tmpdir_factory, build_dep): 'tests_require', 'python_requires', 'install_requires', + 'long_description_content_type', ] assert not match or match.group(1).strip('"\'') in allowed_unknowns From d89682fcba90595d5d6aaf071d6efcc815bceba8 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Mon, 21 Oct 2019 16:49:13 -0700 Subject: [PATCH 17/49] Change coding cookie to use utf-8 (lowercase) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While perfectly valid, the encoding 'UTF-8' (uppercase) is not recognized by the Emacs MULE system. As such, it displays the following warning when opening a file with it used as an encoding cookie: Warning (mule): Invalid coding system ‘UTF-8’ is specified for the current buffer/file by the :coding tag. It is highly recommended to fix it before writing to a file. Some discussion of this can be found at: https://stackoverflow.com/questions/14031724/how-to-make-emacs-accept-utf-8-uppercase-encoding While the post does offer a workaround for Emacs users, rather than ask all to implement it, use the more typical utf-8 (lowercase). --- setuptools/tests/test_config.py | 2 +- setuptools/tests/test_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setuptools/tests/test_config.py b/setuptools/tests/test_config.py index 1b94a58689..69d8d00db3 100644 --- a/setuptools/tests/test_config.py +++ b/setuptools/tests/test_config.py @@ -1,4 +1,4 @@ -# -*- coding: UTF-8 -*- +# -*- coding: utf-8 -*- from __future__ import unicode_literals import contextlib diff --git a/setuptools/tests/test_test.py b/setuptools/tests/test_test.py index faaa6ba90a..3415913b42 100644 --- a/setuptools/tests/test_test.py +++ b/setuptools/tests/test_test.py @@ -1,4 +1,4 @@ -# -*- coding: UTF-8 -*- +# -*- coding: utf-8 -*- from __future__ import unicode_literals From cd84510713ada48bf33d4efa749c2952e3fc1a49 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Sat, 19 Oct 2019 08:39:30 -0700 Subject: [PATCH 18/49] Deprecate the test command Provide a warning to users. Suggest using tox as an alternative generic entry point. Refs #1684 --- changelog.d/1878.change.rst | 1 + docs/setuptools.txt | 13 +++++++++ setuptools/command/test.py | 10 ++++++- setuptools/tests/test_test.py | 50 +++++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 changelog.d/1878.change.rst diff --git a/changelog.d/1878.change.rst b/changelog.d/1878.change.rst new file mode 100644 index 0000000000..0774b5d3e5 --- /dev/null +++ b/changelog.d/1878.change.rst @@ -0,0 +1 @@ +Formally deprecated the ``test`` command, with the recommendation that users migrate to ``tox``. diff --git a/docs/setuptools.txt b/docs/setuptools.txt index 26a3044e82..f69b75c249 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -346,6 +346,8 @@ unless you need the associated ``setuptools`` feature. specified test suite, e.g. via ``setup.py test``. See the section on the `test`_ command below for more details. + New in 41.5.0: Deprecated the test command. + ``tests_require`` If your project's tests need one or more additional packages besides those needed to install it, you can use this option to specify them. It should @@ -357,6 +359,8 @@ unless you need the associated ``setuptools`` feature. are run, but only downloaded to the project's setup directory if they're not already installed locally. + New in 41.5.0: Deprecated the test command. + .. _test_loader: ``test_loader`` @@ -380,6 +384,8 @@ unless you need the associated ``setuptools`` feature. as long as you use the ``tests_require`` option to ensure that the package containing the loader class is available when the ``test`` command is run. + New in 41.5.0: Deprecated the test command. + ``eager_resources`` A list of strings naming resources that should be extracted together, if any of them is needed, or if any C extensions included in the project are @@ -2142,6 +2148,11 @@ distutils configuration file the option will be added to (or removed from). ``test`` - Build package and run a unittest suite ================================================= +.. warning:: + ``test`` is deprecated and will be removed in a future version. Users + looking for a generic test entry point independent of test runner are + encouraged to use `tox `_. + When doing test-driven development, or running automated builds that need testing before they are deployed for downloading or use, it's often useful to be able to run a project's unit tests without actually deploying the project @@ -2187,6 +2198,8 @@ available: If you did not set a ``test_suite`` in your ``setup()`` call, and do not provide a ``--test-suite`` option, an error will occur. +New in 41.5.0: Deprecated the test command. + .. _upload: diff --git a/setuptools/command/test.py b/setuptools/command/test.py index 973e4eb214..c148b38d10 100644 --- a/setuptools/command/test.py +++ b/setuptools/command/test.py @@ -74,7 +74,7 @@ def __get__(self, obj, objtype=None): class test(Command): """Command to run unit tests after in-place build""" - description = "run unit tests after in-place build" + description = "run unit tests after in-place build (deprecated)" user_options = [ ('test-module=', 'm', "Run 'test_suite' in specified module"), @@ -214,6 +214,14 @@ def install_dists(dist): return itertools.chain(ir_d, tr_d, er_d) def run(self): + self.announce( + "WARNING: Testing via this command is deprecated and will be " + "removed in a future version. Users looking for a generic test " + "entry point independent of test runner are encouraged to use " + "tox.", + log.WARN, + ) + installed_dists = self.install_dists(self.distribution) cmd = ' '.join(self._argv) diff --git a/setuptools/tests/test_test.py b/setuptools/tests/test_test.py index faaa6ba90a..382bd640e7 100644 --- a/setuptools/tests/test_test.py +++ b/setuptools/tests/test_test.py @@ -2,6 +2,7 @@ from __future__ import unicode_literals +import mock from distutils import log import os @@ -124,3 +125,52 @@ def test_test(self): cmd.run() out, err = capfd.readouterr() assert out == 'Foo\n' + + +@pytest.mark.usefixtures('sample_test') +def test_warns_deprecation(capfd): + params = dict( + name='foo', + packages=['name', 'name.space', 'name.space.tests'], + namespace_packages=['name'], + test_suite='name.space.tests.test_suite', + use_2to3=True + ) + dist = Distribution(params) + dist.script_name = 'setup.py' + cmd = test(dist) + cmd.ensure_finalized() + cmd.announce = mock.Mock() + cmd.run() + capfd.readouterr() + msg = ( + "WARNING: Testing via this command is deprecated and will be " + "removed in a future version. Users looking for a generic test " + "entry point independent of test runner are encouraged to use " + "tox." + ) + cmd.announce.assert_any_call(msg, log.WARN) + + +@pytest.mark.usefixtures('sample_test') +def test_deprecation_stderr(capfd): + params = dict( + name='foo', + packages=['name', 'name.space', 'name.space.tests'], + namespace_packages=['name'], + test_suite='name.space.tests.test_suite', + use_2to3=True + ) + dist = Distribution(params) + dist.script_name = 'setup.py' + cmd = test(dist) + cmd.ensure_finalized() + cmd.run() + out, err = capfd.readouterr() + msg = ( + "WARNING: Testing via this command is deprecated and will be " + "removed in a future version. Users looking for a generic test " + "entry point independent of test runner are encouraged to use " + "tox.\n" + ) + assert msg in err From 4069e0b536802a47c8c15ce839fd98a5f4c84620 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Mon, 21 Oct 2019 17:01:18 -0700 Subject: [PATCH 19/49] Remove outdated comment and suppressed exception from test_test.py The test command has not called sys.exit since commit 2c4fd43277fc477d85b50e15c37b176136676270. --- setuptools/tests/test_test.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/setuptools/tests/test_test.py b/setuptools/tests/test_test.py index 280c837bf8..6242a018c4 100644 --- a/setuptools/tests/test_test.py +++ b/setuptools/tests/test_test.py @@ -86,9 +86,7 @@ def test_test(capfd): dist.script_name = 'setup.py' cmd = test(dist) cmd.ensure_finalized() - # The test runner calls sys.exit - with contexts.suppress_exceptions(SystemExit): - cmd.run() + cmd.run() out, err = capfd.readouterr() assert out == 'Foo\n' @@ -120,9 +118,7 @@ def test_test(self): dist.script_name = 'setup.py' cmd = test(dist) cmd.ensure_finalized() - # The test runner calls sys.exit - with contexts.suppress_exceptions(SystemExit): - cmd.run() + cmd.run() out, err = capfd.readouterr() assert out == 'Foo\n' From e4ef537525c2b1d120497379da9484c33ea3d773 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Mon, 21 Oct 2019 17:51:26 -0700 Subject: [PATCH 20/49] Add a trove classifier to document support for Python 3.8 --- changelog.d/1884.doc.rst | 1 + setup.cfg | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/1884.doc.rst diff --git a/changelog.d/1884.doc.rst b/changelog.d/1884.doc.rst new file mode 100644 index 0000000000..45615d5d88 --- /dev/null +++ b/changelog.d/1884.doc.rst @@ -0,0 +1 @@ +Added a trove classifier to document support for Python 3.8. diff --git a/setup.cfg b/setup.cfg index a55106a5a4..373ae8b535 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,7 @@ classifiers = Programming Language :: Python :: 3.5 Programming Language :: Python :: 3.6 Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 Topic :: Software Development :: Libraries :: Python Modules Topic :: System :: Archiving :: Packaging Topic :: System :: Systems Administration From 89f15f448679e2ebedb40ed348423596dc658d31 Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Mon, 21 Oct 2019 17:51:26 -0700 Subject: [PATCH 21/49] Add Python 3.8 final to Travis test matrix --- .travis.yml | 3 ++- changelog.d/1886.misc.rst | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 changelog.d/1886.misc.rst diff --git a/.travis.yml b/.travis.yml index 8b7cece8b4..7088d16621 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,9 @@ jobs: - python: 3.5 - &default_py python: 3.6 + - python: 3.7 - &latest_py3 - python: 3.7 + python: 3.8 - <<: *latest_py3 env: LANG=C - python: 3.8-dev diff --git a/changelog.d/1886.misc.rst b/changelog.d/1886.misc.rst new file mode 100644 index 0000000000..5e3f2873a8 --- /dev/null +++ b/changelog.d/1886.misc.rst @@ -0,0 +1 @@ +Added Python 3.8 release to the Travis test matrix. From 19eb6bf8bd28f1b9b66288c797d67eb8da71508d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 27 Oct 2019 18:26:59 -0400 Subject: [PATCH 22/49] Fix typo --- docs/easy_install.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/easy_install.txt b/docs/easy_install.txt index e247d8fd76..544b9efd59 100644 --- a/docs/easy_install.txt +++ b/docs/easy_install.txt @@ -317,7 +317,7 @@ Note that instead of changing your ``PATH`` to include the Python scripts directory, you can also retarget the installation location for scripts so they go on a directory that's already on the ``PATH``. For more information see `Command-Line Options`_ and `Configuration Files`_. During installation, -pass command line options (such as ``--script-dir``) to to control where +pass command line options (such as ``--script-dir``) to control where scripts will be installed. From 6e8b1da1fb146186c52d8bdec5af969f26532ece Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 27 Oct 2019 18:42:56 -0400 Subject: [PATCH 23/49] =?UTF-8?q?Bump=20version:=2041.4.0=20=E2=86=92=2041?= =?UTF-8?q?.5.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- CHANGES.rst | 14 ++++++++++++++ changelog.d/1811.change.rst | 1 - changelog.d/1814.change.rst | 1 - changelog.d/1824.change.rst | 1 - changelog.d/1860.doc.rst | 1 - changelog.d/1862.doc.rst | 1 - changelog.d/1868.doc.rst | 1 - changelog.d/1878.change.rst | 1 - changelog.d/1884.doc.rst | 1 - changelog.d/1886.misc.rst | 1 - setup.cfg | 2 +- 12 files changed, 16 insertions(+), 11 deletions(-) delete mode 100644 changelog.d/1811.change.rst delete mode 100644 changelog.d/1814.change.rst delete mode 100644 changelog.d/1824.change.rst delete mode 100644 changelog.d/1860.doc.rst delete mode 100644 changelog.d/1862.doc.rst delete mode 100644 changelog.d/1868.doc.rst delete mode 100644 changelog.d/1878.change.rst delete mode 100644 changelog.d/1884.doc.rst delete mode 100644 changelog.d/1886.misc.rst diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d6768cb8a8..aeefc8f2c6 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 41.4.0 +current_version = 41.5.0 commit = True tag = True diff --git a/CHANGES.rst b/CHANGES.rst index ecde25a571..7e9fc0f6e8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,17 @@ +v41.5.0 +------- + +* #1811: Improve Visual C++ 14.X support, mainly for Visual Studio 2017 and 2019. +* #1814: Fix ``pkg_resources.Requirement`` hash/equality implementation: take PEP 508 direct URL into account. +* #1824: Fix tests when running under ``python3.10``. +* #1878: Formally deprecated the ``test`` command, with the recommendation that users migrate to ``tox``. +* #1860: Update documentation to mention the egg format is not supported by pip and dependency links support was dropped starting with pip 19.0. +* #1862: Drop ez_setup documentation: deprecated for some time (last updated in 2016), and still relying on easy_install (deprecated too). +* #1868: Drop most documentation references to (deprecated) EasyInstall. +* #1884: Added a trove classifier to document support for Python 3.8. +* #1886: Added Python 3.8 release to the Travis test matrix. + + v41.4.0 ------- diff --git a/changelog.d/1811.change.rst b/changelog.d/1811.change.rst deleted file mode 100644 index dc52c6dbbe..0000000000 --- a/changelog.d/1811.change.rst +++ /dev/null @@ -1 +0,0 @@ -Improve Visual C++ 14.X support, mainly for Visual Studio 2017 and 2019. \ No newline at end of file diff --git a/changelog.d/1814.change.rst b/changelog.d/1814.change.rst deleted file mode 100644 index c936699dd8..0000000000 --- a/changelog.d/1814.change.rst +++ /dev/null @@ -1 +0,0 @@ -Fix ``pkg_resources.Requirement`` hash/equality implementation: take PEP 508 direct URL into account. diff --git a/changelog.d/1824.change.rst b/changelog.d/1824.change.rst deleted file mode 100644 index 5f60903687..0000000000 --- a/changelog.d/1824.change.rst +++ /dev/null @@ -1 +0,0 @@ -Fix tests when running under ``python3.10``. diff --git a/changelog.d/1860.doc.rst b/changelog.d/1860.doc.rst deleted file mode 100644 index f3554643fa..0000000000 --- a/changelog.d/1860.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Update documentation to mention the egg format is not supported by pip and dependency links support was dropped starting with pip 19.0. diff --git a/changelog.d/1862.doc.rst b/changelog.d/1862.doc.rst deleted file mode 100644 index b71583ba37..0000000000 --- a/changelog.d/1862.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Drop ez_setup documentation: deprecated for some time (last updated in 2016), and still relying on easy_install (deprecated too). diff --git a/changelog.d/1868.doc.rst b/changelog.d/1868.doc.rst deleted file mode 100644 index 82283d7aea..0000000000 --- a/changelog.d/1868.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Drop most documentation references to (deprecated) EasyInstall. diff --git a/changelog.d/1878.change.rst b/changelog.d/1878.change.rst deleted file mode 100644 index 0774b5d3e5..0000000000 --- a/changelog.d/1878.change.rst +++ /dev/null @@ -1 +0,0 @@ -Formally deprecated the ``test`` command, with the recommendation that users migrate to ``tox``. diff --git a/changelog.d/1884.doc.rst b/changelog.d/1884.doc.rst deleted file mode 100644 index 45615d5d88..0000000000 --- a/changelog.d/1884.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Added a trove classifier to document support for Python 3.8. diff --git a/changelog.d/1886.misc.rst b/changelog.d/1886.misc.rst deleted file mode 100644 index 5e3f2873a8..0000000000 --- a/changelog.d/1886.misc.rst +++ /dev/null @@ -1 +0,0 @@ -Added Python 3.8 release to the Travis test matrix. diff --git a/setup.cfg b/setup.cfg index 373ae8b535..0e030cde82 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,7 +19,7 @@ universal = 1 [metadata] name = setuptools -version = 41.4.0 +version = 41.5.0 description = Easily download, build, install, upgrade, and uninstall Python packages author = Python Packaging Authority author_email = distutils-sig@python.org From f430e585d84a5c63bb3b52e17af2f1b40fec8b71 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 27 Oct 2019 19:16:36 -0400 Subject: [PATCH 24/49] Remove apparently unrelated change to test --- setuptools/tests/test_integration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setuptools/tests/test_integration.py b/setuptools/tests/test_integration.py index 1c0b2b18bb..1e13218899 100644 --- a/setuptools/tests/test_integration.py +++ b/setuptools/tests/test_integration.py @@ -143,7 +143,6 @@ def test_build_deps_on_distutils(request, tmpdir_factory, build_dep): 'tests_require', 'python_requires', 'install_requires', - 'long_description_content_type', ] assert not match or match.group(1).strip('"\'') in allowed_unknowns From 85a9ca5e75abf00e0dde55dde4e2b0a11f93c04a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 27 Oct 2019 19:50:53 -0400 Subject: [PATCH 25/49] Extract 'imp' re-implementation to setuptools._imp and wrap it in py27compat for compatibility. --- conftest.py | 1 + setuptools/_imp.py | 72 ++++++++++++++++++++++++++++++ setuptools/depends.py | 95 ++++------------------------------------ setuptools/py27compat.py | 32 ++++++++++++++ 4 files changed, 113 insertions(+), 87 deletions(-) create mode 100644 setuptools/_imp.py diff --git a/conftest.py b/conftest.py index 0d7b274c07..1746bfb588 100644 --- a/conftest.py +++ b/conftest.py @@ -19,6 +19,7 @@ def pytest_addoption(parser): if sys.version_info < (3,): collect_ignore.append('setuptools/lib2to3_ex.py') + collect_ignore.append('setuptools/_imp.py') if sys.version_info < (3, 6): diff --git a/setuptools/_imp.py b/setuptools/_imp.py new file mode 100644 index 0000000000..6bc9024368 --- /dev/null +++ b/setuptools/_imp.py @@ -0,0 +1,72 @@ +""" +Re-implementation of find_module and get_frozen_object +from the deprecated imp module. +""" + +import os +import importlib.util +import importlib.machinery + + +PY_SOURCE = 1 +PY_COMPILED = 2 +C_EXTENSION = 3 +C_BUILTIN = 6 +PY_FROZEN = 7 + + +def find_module(module, paths=None): + """ + """ + spec = importlib.util.find_spec(module, paths) + if spec is None: + raise ImportError("Can't find %s" % module) + if not spec.has_location and hasattr(spec, 'submodule_search_locations'): + spec = importlib.util.spec_from_loader('__init__.py', spec.loader) + + kind = -1 + file = None + static = isinstance(spec.loader, type) + if spec.origin == 'frozen' or static and issubclass( + spec.loader, importlib.machinery.FrozenImporter): + kind = PY_FROZEN + path = None # imp compabilty + suffix = mode = '' # imp compability + elif spec.origin == 'built-in' or static and issubclass( + spec.loader, importlib.machinery.BuiltinImporter): + kind = C_BUILTIN + path = None # imp compabilty + suffix = mode = '' # imp compability + elif spec.has_location: + path = spec.origin + suffix = os.path.splitext(path)[1] + mode = 'r' if suffix in importlib.machinery.SOURCE_SUFFIXES else 'rb' + + if suffix in importlib.machinery.SOURCE_SUFFIXES: + kind = PY_SOURCE + elif suffix in importlib.machinery.BYTECODE_SUFFIXES: + kind = PY_COMPILED + elif suffix in importlib.machinery.EXTENSION_SUFFIXES: + kind = C_EXTENSION + + if kind in {PY_SOURCE, PY_COMPILED}: + file = open(path, mode) + else: + path = None + suffix = mode = '' + + return file, path, (suffix, mode, kind) + + +def get_frozen_object(module, paths): + spec = importlib.util.find_spec(module, paths) + if hasattr(spec, 'submodule_search_locations'): + spec = importlib.util.spec_from_loader('__init__.py', spec.loader) + return spec.loader.get_code(module) + + +def get_module(module, paths, info): + spec = importlib.util.find_spec(module, paths) + if hasattr(spec, 'submodule_search_locations'): + spec = importlib.util.spec_from_loader('__init__.py', spec.loader) + return importlib.util.module_from_spec(spec) diff --git a/setuptools/depends.py b/setuptools/depends.py index 97f0ed9dae..eed4913a6b 100644 --- a/setuptools/depends.py +++ b/setuptools/depends.py @@ -1,22 +1,11 @@ import sys import marshal from distutils.version import StrictVersion -from setuptools.extern import six from .py33compat import Bytecode -if six.PY2: - import imp - from imp import PKG_DIRECTORY, PY_COMPILED, PY_SOURCE, PY_FROZEN -else: - import os.path - from importlib.util import find_spec, spec_from_loader - from importlib.machinery import SOURCE_SUFFIXES, BYTECODE_SUFFIXES, EXTENSION_SUFFIXES, BuiltinImporter, FrozenImporter - PY_SOURCE = 1 - PY_COMPILED = 2 - C_EXTENSION = 3 - C_BUILTIN = 6 - PY_FROZEN = 7 +from .py27compat import find_module, PY_COMPILED, PY_FROZEN, PY_SOURCE +from . import py27compat __all__ = [ @@ -27,7 +16,8 @@ class Require: """A prerequisite to building or installing a distribution""" - def __init__(self, name, requested_version, module, homepage='', + def __init__( + self, name, requested_version, module, homepage='', attribute=None, format=None): if format is None and requested_version is not None: @@ -91,63 +81,6 @@ def is_current(self, paths=None): return self.version_ok(version) -def find_module(module, paths=None): - """Just like 'imp.find_module()', but with package support""" - if six.PY3: - spec = find_spec(module, paths) - if spec is None: - raise ImportError("Can't find %s" % module) - if not spec.has_location and hasattr(spec, 'submodule_search_locations'): - spec = spec_from_loader('__init__.py', spec.loader) - - kind = -1 - file = None - static = isinstance(spec.loader, type) - if spec.origin == 'frozen' or static and issubclass(spec.loader, FrozenImporter): - kind = PY_FROZEN - path = None # imp compabilty - suffix = mode = '' # imp compability - elif spec.origin == 'built-in' or static and issubclass(spec.loader, BuiltinImporter): - kind = C_BUILTIN - path = None # imp compabilty - suffix = mode = '' # imp compability - elif spec.has_location: - frozen = False - path = spec.origin - suffix = os.path.splitext(path)[1] - mode = 'r' if suffix in SOURCE_SUFFIXES else 'rb' - - if suffix in SOURCE_SUFFIXES: - kind = PY_SOURCE - elif suffix in BYTECODE_SUFFIXES: - kind = PY_COMPILED - elif suffix in EXTENSION_SUFFIXES: - kind = C_EXTENSION - - if kind in {PY_SOURCE, PY_COMPILED}: - file = open(path, mode) - else: - path = None - suffix = mode= '' - - return file, path, (suffix, mode, kind) - - else: - parts = module.split('.') - while parts: - part = parts.pop(0) - f, path, (suffix, mode, kind) = info = imp.find_module(part, paths) - - if kind == PKG_DIRECTORY: - parts = parts or ['__init__'] - paths = [path] - - elif parts: - raise ImportError("Can't find %r in %s" % (parts, module)) - - return info - - def get_module_constant(module, symbol, default=-1, paths=None): """Find 'module' by searching 'paths', and extract 'symbol' @@ -156,35 +89,23 @@ def get_module_constant(module, symbol, default=-1, paths=None): constant. Otherwise, return 'default'.""" try: - f, path, (suffix, mode, kind) = find_module(module, paths) + f, path, (suffix, mode, kind) = info = find_module(module, paths) except ImportError: # Module doesn't exist return None - if six.PY3: - spec = find_spec(module, paths) - if hasattr(spec, 'submodule_search_locations'): - spec = spec_from_loader('__init__.py', spec.loader) - try: if kind == PY_COMPILED: f.read(8) # skip magic & date code = marshal.load(f) elif kind == PY_FROZEN: - if six.PY2: - code = imp.get_frozen_object(module) - else: - code = spec.loader.get_code(module) + code = py27compat.get_frozen_object(module, paths) elif kind == PY_SOURCE: code = compile(f.read(), path, 'exec') else: # Not something we can parse; we'll have to import it. :( - if module not in sys.modules: - if six.PY2: - imp.load_module(module, f, path, (suffix, mode, kind)) - else: - sys.modules[module] = module_from_spec(spec) - return getattr(sys.modules[module], symbol, None) + imported = py27compat.get_module(module, paths, info) + return getattr(imported, symbol, None) finally: if f: diff --git a/setuptools/py27compat.py b/setuptools/py27compat.py index 2985011b92..cf5fb33efe 100644 --- a/setuptools/py27compat.py +++ b/setuptools/py27compat.py @@ -2,6 +2,7 @@ Compatibility Support for Python 2.7 and earlier """ +import sys import platform from setuptools.extern import six @@ -26,3 +27,34 @@ def get_all_headers(message, key): rmtree_safe = str if linux_py2_ascii else lambda x: x """Workaround for http://bugs.python.org/issue24672""" + + +try: + from ._imp import find_module, PY_COMPILED, PY_FROZEN, PY_SOURCE + from ._imp import get_frozen_object, get_module +except ImportError: + import imp + from imp import PY_COMPILED, PY_FROZEN, PY_SOURCE # noqa + + def find_module(module, paths=None): + """Just like 'imp.find_module()', but with package support""" + parts = module.split('.') + while parts: + part = parts.pop(0) + f, path, (suffix, mode, kind) = info = imp.find_module(part, paths) + + if kind == imp.PKG_DIRECTORY: + parts = parts or ['__init__'] + paths = [path] + + elif parts: + raise ImportError("Can't find %r in %s" % (parts, module)) + + return info + + def get_frozen_object(module, paths): + return imp.get_frozen_object(module) + + def get_module(module, paths, info): + imp.load_module(*info) + return sys.modules[module] From 773f1ec3c2622a78ee0280eb1d2b03c60871c52b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 27 Oct 2019 19:54:42 -0400 Subject: [PATCH 26/49] Rely on contextlib.closing for brevity. --- setuptools/depends.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/setuptools/depends.py b/setuptools/depends.py index eed4913a6b..a37675cbd9 100644 --- a/setuptools/depends.py +++ b/setuptools/depends.py @@ -1,5 +1,6 @@ import sys import marshal +import contextlib from distutils.version import StrictVersion from .py33compat import Bytecode @@ -81,6 +82,17 @@ def is_current(self, paths=None): return self.version_ok(version) +def maybe_close(f): + @contextlib.contextmanager + def empty(): + yield + return + if not f: + return empty() + + return contextlib.closing(f) + + def get_module_constant(module, symbol, default=-1, paths=None): """Find 'module' by searching 'paths', and extract 'symbol' @@ -94,7 +106,7 @@ def get_module_constant(module, symbol, default=-1, paths=None): # Module doesn't exist return None - try: + with maybe_close(f): if kind == PY_COMPILED: f.read(8) # skip magic & date code = marshal.load(f) @@ -107,10 +119,6 @@ def get_module_constant(module, symbol, default=-1, paths=None): imported = py27compat.get_module(module, paths, info) return getattr(imported, symbol, None) - finally: - if f: - f.close() - return extract_constant(code, symbol, default) From 37d617cd983446dfe8073038d03dd31fc7f66796 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 27 Oct 2019 20:40:07 -0400 Subject: [PATCH 27/49] Extract _resolve --- setuptools/_imp.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/setuptools/_imp.py b/setuptools/_imp.py index 6bc9024368..c400d455c2 100644 --- a/setuptools/_imp.py +++ b/setuptools/_imp.py @@ -60,13 +60,17 @@ def find_module(module, paths=None): def get_frozen_object(module, paths): spec = importlib.util.find_spec(module, paths) - if hasattr(spec, 'submodule_search_locations'): - spec = importlib.util.spec_from_loader('__init__.py', spec.loader) - return spec.loader.get_code(module) + return spec.loader.get_code(_resolve(module)) + + +def _resolve(spec): + return ( + importlib.util.spec_from_loader('__init__.py', spec.loader) + if hasattr(spec, 'submodule_search_locations') + else spec + ) def get_module(module, paths, info): spec = importlib.util.find_spec(module, paths) - if hasattr(spec, 'submodule_search_locations'): - spec = importlib.util.spec_from_loader('__init__.py', spec.loader) - return importlib.util.module_from_spec(spec) + return importlib.util.module_from_spec(_resolve(spec)) From 4948b14a66f00eba749e52b77281f711413e071b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 27 Oct 2019 20:40:58 -0400 Subject: [PATCH 28/49] Avoid _resolve in get_module (causes failures). --- setuptools/_imp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setuptools/_imp.py b/setuptools/_imp.py index c400d455c2..cee9155145 100644 --- a/setuptools/_imp.py +++ b/setuptools/_imp.py @@ -73,4 +73,4 @@ def _resolve(spec): def get_module(module, paths, info): spec = importlib.util.find_spec(module, paths) - return importlib.util.module_from_spec(_resolve(spec)) + return importlib.util.module_from_spec(spec) From 16051d6b2d3641dc8951e90f7f04bcd04b8954d3 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 28 Oct 2019 10:12:53 +0300 Subject: [PATCH 29/49] imp load_module fix --- setuptools/py27compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setuptools/py27compat.py b/setuptools/py27compat.py index cf5fb33efe..1d57360f4e 100644 --- a/setuptools/py27compat.py +++ b/setuptools/py27compat.py @@ -56,5 +56,5 @@ def get_frozen_object(module, paths): return imp.get_frozen_object(module) def get_module(module, paths, info): - imp.load_module(*info) + imp.load_module(module, *info) return sys.modules[module] From 65fe7abeaba9e82b8f3755054759fed21c0c489b Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 28 Oct 2019 10:28:55 +0300 Subject: [PATCH 30/49] py34 compat --- setuptools/_imp.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setuptools/_imp.py b/setuptools/_imp.py index cee9155145..09073d44fe 100644 --- a/setuptools/_imp.py +++ b/setuptools/_imp.py @@ -4,6 +4,7 @@ """ import os +import sys import importlib.util import importlib.machinery @@ -70,7 +71,12 @@ def _resolve(spec): else spec ) +def _module_from_spec(spec): + if sys.version_info >= (3, 5): + return importlib.util.module_from_spec(spec) + else: + return spec.loader.load_module(spec.name) def get_module(module, paths, info): spec = importlib.util.find_spec(module, paths) - return importlib.util.module_from_spec(spec) + return _module_from_spec(spec) From ce01c0199f93612848e664bfd920083168eaa293 Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 28 Oct 2019 10:59:21 +0300 Subject: [PATCH 31/49] add docstring to find_module --- setuptools/_imp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setuptools/_imp.py b/setuptools/_imp.py index 09073d44fe..dc4b1b2c5d 100644 --- a/setuptools/_imp.py +++ b/setuptools/_imp.py @@ -17,8 +17,7 @@ def find_module(module, paths=None): - """ - """ + """Just like 'imp.find_module()', but with package support""" spec = importlib.util.find_spec(module, paths) if spec is None: raise ImportError("Can't find %s" % module) From 2f4952927e41643100e6e6f58124f34331c14add Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Mon, 28 Oct 2019 11:07:27 +0300 Subject: [PATCH 32/49] remove _resolve --- setuptools/_imp.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/setuptools/_imp.py b/setuptools/_imp.py index dc4b1b2c5d..49ddc852be 100644 --- a/setuptools/_imp.py +++ b/setuptools/_imp.py @@ -63,13 +63,6 @@ def get_frozen_object(module, paths): return spec.loader.get_code(_resolve(module)) -def _resolve(spec): - return ( - importlib.util.spec_from_loader('__init__.py', spec.loader) - if hasattr(spec, 'submodule_search_locations') - else spec - ) - def _module_from_spec(spec): if sys.version_info >= (3, 5): return importlib.util.module_from_spec(spec) From 7489ea4047661a7dbd46ff155abe45c548284676 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Mon, 28 Oct 2019 18:47:31 +0100 Subject: [PATCH 33/49] msvc: fix Python 2 support --- changelog.d/1891.change.rst | 1 + setuptools/msvc.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/1891.change.rst diff --git a/changelog.d/1891.change.rst b/changelog.d/1891.change.rst new file mode 100644 index 0000000000..81ccff10e4 --- /dev/null +++ b/changelog.d/1891.change.rst @@ -0,0 +1 @@ +Fix code for detecting Visual Studio's version on Windows under Python 2. diff --git a/setuptools/msvc.py b/setuptools/msvc.py index ffa7053b64..2ffe1c81ee 100644 --- a/setuptools/msvc.py +++ b/setuptools/msvc.py @@ -20,6 +20,7 @@ """ import json +from io import open from os import listdir, pathsep from os.path import join, isfile, isdir, dirname import sys From 7748921de342160ca2dc9c9539562bb9c924e14c Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Mon, 28 Oct 2019 19:12:39 +0100 Subject: [PATCH 34/49] =?UTF-8?q?Bump=20version:=2041.5.0=20=E2=86=92=2041?= =?UTF-8?q?.5.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- CHANGES.rst | 6 ++++++ changelog.d/1891.change.rst | 1 - setup.cfg | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/1891.change.rst diff --git a/.bumpversion.cfg b/.bumpversion.cfg index aeefc8f2c6..0dc75e9bfd 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 41.5.0 +current_version = 41.5.1 commit = True tag = True diff --git a/CHANGES.rst b/CHANGES.rst index 7e9fc0f6e8..bac356385f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v41.5.1 +------- + +* #1891: Fix code for detecting Visual Studio's version on Windows under Python 2. + + v41.5.0 ------- diff --git a/changelog.d/1891.change.rst b/changelog.d/1891.change.rst deleted file mode 100644 index 81ccff10e4..0000000000 --- a/changelog.d/1891.change.rst +++ /dev/null @@ -1 +0,0 @@ -Fix code for detecting Visual Studio's version on Windows under Python 2. diff --git a/setup.cfg b/setup.cfg index 0e030cde82..8038b463cc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,7 +19,7 @@ universal = 1 [metadata] name = setuptools -version = 41.5.0 +version = 41.5.1 description = Easily download, build, install, upgrade, and uninstall Python packages author = Python Packaging Authority author_email = distutils-sig@python.org From 3a0520b43dfac9f6ba507c6d09a60290219a0802 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 28 Oct 2019 21:52:40 -0400 Subject: [PATCH 35/49] Extract compatibility function into compatibility module. --- setuptools/_imp.py | 10 +++------- setuptools/py34compat.py | 8 ++++++++ 2 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 setuptools/py34compat.py diff --git a/setuptools/_imp.py b/setuptools/_imp.py index 49ddc852be..ee719c9a36 100644 --- a/setuptools/_imp.py +++ b/setuptools/_imp.py @@ -8,6 +8,8 @@ import importlib.util import importlib.machinery +from .py34compat import module_from_spec + PY_SOURCE = 1 PY_COMPILED = 2 @@ -63,12 +65,6 @@ def get_frozen_object(module, paths): return spec.loader.get_code(_resolve(module)) -def _module_from_spec(spec): - if sys.version_info >= (3, 5): - return importlib.util.module_from_spec(spec) - else: - return spec.loader.load_module(spec.name) - def get_module(module, paths, info): spec = importlib.util.find_spec(module, paths) - return _module_from_spec(spec) + return module_from_spec(spec) diff --git a/setuptools/py34compat.py b/setuptools/py34compat.py new file mode 100644 index 0000000000..bc7eefa99f --- /dev/null +++ b/setuptools/py34compat.py @@ -0,0 +1,8 @@ +import importlib.util + + +try: + module_from_spec = importlib.util.module_from_spec +except AttributeError: + def module_from_spec(spec): + return spec.loader.load_module(spec.name) From 2175d6bdcf4fe626e713961bb0315c91f206746b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 28 Oct 2019 21:57:31 -0400 Subject: [PATCH 36/49] Add changelog entry. --- changelog.d/479.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/479.bugfix.rst diff --git a/changelog.d/479.bugfix.rst b/changelog.d/479.bugfix.rst new file mode 100644 index 0000000000..3c33964edb --- /dev/null +++ b/changelog.d/479.bugfix.rst @@ -0,0 +1 @@ +Replace usage of deprecated ``imp`` module with local re-implementation in ``setuptools._imp``. From e1f340b53f0088993b16e19999a4d6b0e86a9991 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 28 Oct 2019 22:01:34 -0400 Subject: [PATCH 37/49] Avoid importerror on older Pythons --- setuptools/py34compat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setuptools/py34compat.py b/setuptools/py34compat.py index bc7eefa99f..54157a6375 100644 --- a/setuptools/py34compat.py +++ b/setuptools/py34compat.py @@ -1,4 +1,4 @@ -import importlib.util +import importlib try: From 82689e1aa8e6548f26f2ce3bcd069411cb39bfcf Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 28 Oct 2019 22:06:44 -0400 Subject: [PATCH 38/49] Ensure importlib.util is imported on Python 3.5 --- setuptools/py34compat.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setuptools/py34compat.py b/setuptools/py34compat.py index 54157a6375..3ad917222a 100644 --- a/setuptools/py34compat.py +++ b/setuptools/py34compat.py @@ -1,5 +1,10 @@ import importlib +try: + import importlib.util +except ImportError: + pass + try: module_from_spec = importlib.util.module_from_spec From 20d6407aa5f68dbeba61e8967290f2fbde4f85ab Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 29 Oct 2019 10:51:54 +0300 Subject: [PATCH 39/49] Allow calling get_frozen_object without paths, raise ImportError when it cant find module --- setuptools/_imp.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/setuptools/_imp.py b/setuptools/_imp.py index ee719c9a36..ab29ef2123 100644 --- a/setuptools/_imp.py +++ b/setuptools/_imp.py @@ -60,11 +60,15 @@ def find_module(module, paths=None): return file, path, (suffix, mode, kind) -def get_frozen_object(module, paths): +def get_frozen_object(module, paths=None): spec = importlib.util.find_spec(module, paths) - return spec.loader.get_code(_resolve(module)) + if not spec: + raise ImportError("Can't find %s" % module) + return spec.loader.get_code(module) def get_module(module, paths, info): spec = importlib.util.find_spec(module, paths) + if not spec: + raise ImportError("Can't find %s" % module) return module_from_spec(spec) From cfa9245a7dea8a35f11580c0bfd27472a3182c7e Mon Sep 17 00:00:00 2001 From: Batuhan Taskaya Date: Tue, 29 Oct 2019 10:53:40 +0300 Subject: [PATCH 40/49] Remove 'sys' import --- setuptools/_imp.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setuptools/_imp.py b/setuptools/_imp.py index ab29ef2123..a3cce9b284 100644 --- a/setuptools/_imp.py +++ b/setuptools/_imp.py @@ -4,7 +4,6 @@ """ import os -import sys import importlib.util import importlib.machinery From 6defe6b8fd6008f61ce5ecfae91dc1df5e123694 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 29 Oct 2019 09:52:32 -0400 Subject: [PATCH 41/49] Rename changelog file --- changelog.d/{479.bugfix.rst => 479.change.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename changelog.d/{479.bugfix.rst => 479.change.rst} (100%) diff --git a/changelog.d/479.bugfix.rst b/changelog.d/479.change.rst similarity index 100% rename from changelog.d/479.bugfix.rst rename to changelog.d/479.change.rst From a0fe403c141369defacf12dccbdc01634bbcb1da Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 29 Oct 2019 09:53:23 -0400 Subject: [PATCH 42/49] =?UTF-8?q?Bump=20version:=2041.5.1=20=E2=86=92=2041?= =?UTF-8?q?.6.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- CHANGES.rst | 6 ++++++ changelog.d/479.change.rst | 1 - setup.cfg | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 changelog.d/479.change.rst diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 0dc75e9bfd..40db5b03f1 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 41.5.1 +current_version = 41.6.0 commit = True tag = True diff --git a/CHANGES.rst b/CHANGES.rst index bac356385f..ba7b4647c8 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -1,3 +1,9 @@ +v41.6.0 +------- + +* #479: Replace usage of deprecated ``imp`` module with local re-implementation in ``setuptools._imp``. + + v41.5.1 ------- diff --git a/changelog.d/479.change.rst b/changelog.d/479.change.rst deleted file mode 100644 index 3c33964edb..0000000000 --- a/changelog.d/479.change.rst +++ /dev/null @@ -1 +0,0 @@ -Replace usage of deprecated ``imp`` module with local re-implementation in ``setuptools._imp``. diff --git a/setup.cfg b/setup.cfg index 8038b463cc..42a3d86c6c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,7 +19,7 @@ universal = 1 [metadata] name = setuptools -version = 41.5.1 +version = 41.6.0 description = Easily download, build, install, upgrade, and uninstall Python packages author = Python Packaging Authority author_email = distutils-sig@python.org From 9fd54518fcb660d9d3f92b1bb242082f20c69c1f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 31 Oct 2019 15:00:55 -0400 Subject: [PATCH 43/49] Suppress deprecation of bdist_wininst. Ref #1823. --- pytest.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pytest.ini b/pytest.ini index 612fb91f63..549e4d68f8 100644 --- a/pytest.ini +++ b/pytest.ini @@ -5,3 +5,6 @@ flake8-ignore = setuptools/site-patch.py F821 setuptools/py*compat.py F811 doctest_optionflags=ELLIPSIS ALLOW_UNICODE +filterwarnings = + # https://github.com/pypa/setuptools/issues/1823 + ignore:distutils.command.bdist_wininst::command is deprecated From 6ac7b4ee036ef8e6954198689d214e8ee9b29118 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 31 Oct 2019 15:58:45 -0400 Subject: [PATCH 44/49] Suppress deprecation of bdist_wininst (redo). Ref #1823. --- pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 549e4d68f8..0370f7f827 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,4 +7,4 @@ flake8-ignore = doctest_optionflags=ELLIPSIS ALLOW_UNICODE filterwarnings = # https://github.com/pypa/setuptools/issues/1823 - ignore:distutils.command.bdist_wininst::command is deprecated + ignore:bdist_wininst command is deprecated From f413f95e95b34b26d9ed9d9c43b3e4b3d30caecc Mon Sep 17 00:00:00 2001 From: Paul Ganssle Date: Thu, 31 Oct 2019 11:25:57 -0400 Subject: [PATCH 45/49] Remove "upload" and "register" commands. The upload and register commands were deprecated over a year ago, in July 2018 (PR GH-1410, discussed in issue GH-1381). It is time to actively remove them in favor of twine. --- changelog.d/1898.breaking.rst | 1 + docs/setuptools.txt | 15 +-- setuptools/command/__init__.py | 3 +- setuptools/command/register.py | 22 ++-- setuptools/command/upload.py | 195 ++------------------------- setuptools/errors.py | 16 +++ setuptools/tests/test_register.py | 43 ++---- setuptools/tests/test_upload.py | 211 ++---------------------------- 8 files changed, 64 insertions(+), 442 deletions(-) create mode 100644 changelog.d/1898.breaking.rst create mode 100644 setuptools/errors.py diff --git a/changelog.d/1898.breaking.rst b/changelog.d/1898.breaking.rst new file mode 100644 index 0000000000..844a8a42fa --- /dev/null +++ b/changelog.d/1898.breaking.rst @@ -0,0 +1 @@ +Removed the "upload" and "register" commands in favor of `twine `_. diff --git a/docs/setuptools.txt b/docs/setuptools.txt index 344ea5bc36..399a56d3ca 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -2087,16 +2087,13 @@ New in 41.5.0: Deprecated the test command. ``upload`` - Upload source and/or egg distributions to PyPI =========================================================== -.. warning:: - **upload** is deprecated in favor of using `twine - `_ - -The ``upload`` command is implemented and `documented -`_ -in distutils. +The ``upload`` command was deprecated in version 40.0 and removed in version +42.0. Use `twine `_ instead. -New in 20.1: Added keyring support. -New in 40.0: Deprecated the upload command. +For more information on the current best practices in uploading your packages +to PyPI, see the Python Packaging User Guide's "Packaging Python Projects" +tutorial specifically the section on `uploading the distribution archives +`_. ----------------------------------------- diff --git a/setuptools/command/__init__.py b/setuptools/command/__init__.py index fe619e2e67..743f5588fa 100644 --- a/setuptools/command/__init__.py +++ b/setuptools/command/__init__.py @@ -2,8 +2,7 @@ 'alias', 'bdist_egg', 'bdist_rpm', 'build_ext', 'build_py', 'develop', 'easy_install', 'egg_info', 'install', 'install_lib', 'rotate', 'saveopts', 'sdist', 'setopt', 'test', 'install_egg_info', 'install_scripts', - 'register', 'bdist_wininst', 'upload_docs', 'upload', 'build_clib', - 'dist_info', + 'bdist_wininst', 'upload_docs', 'build_clib', 'dist_info', ] from distutils.command.bdist import bdist diff --git a/setuptools/command/register.py b/setuptools/command/register.py index 98bc01566f..b8266b9a60 100644 --- a/setuptools/command/register.py +++ b/setuptools/command/register.py @@ -1,18 +1,18 @@ from distutils import log import distutils.command.register as orig +from setuptools.errors import RemovedCommandError + class register(orig.register): - __doc__ = orig.register.__doc__ + """Formerly used to register packages on PyPI.""" def run(self): - try: - # Make sure that we are using valid current name/version info - self.run_command('egg_info') - orig.register.run(self) - finally: - self.announce( - "WARNING: Registering is deprecated, use twine to " - "upload instead (https://pypi.org/p/twine/)", - log.WARN - ) + msg = ( + "The register command has been removed, use twine to upload " + + "instead (https://pypi.org/p/twine)" + ) + + self.announce("ERROR: " + msg, log.ERROR) + + raise RemovedCommandError(msg) diff --git a/setuptools/command/upload.py b/setuptools/command/upload.py index 6db8888bb2..ec7f81e227 100644 --- a/setuptools/command/upload.py +++ b/setuptools/command/upload.py @@ -1,196 +1,17 @@ -import io -import os -import hashlib -import getpass - -from base64 import standard_b64encode - from distutils import log from distutils.command import upload as orig -from distutils.spawn import spawn - -from distutils.errors import DistutilsError -from setuptools.extern.six.moves.urllib.request import urlopen, Request -from setuptools.extern.six.moves.urllib.error import HTTPError -from setuptools.extern.six.moves.urllib.parse import urlparse +from setuptools.errors import RemovedCommandError class upload(orig.upload): - """ - Override default upload behavior to obtain password - in a variety of different ways. - """ - def run(self): - try: - orig.upload.run(self) - finally: - self.announce( - "WARNING: Uploading via this command is deprecated, use twine " - "to upload instead (https://pypi.org/p/twine/)", - log.WARN - ) + """Formerly used to upload packages to PyPI.""" - def finalize_options(self): - orig.upload.finalize_options(self) - self.username = ( - self.username or - getpass.getuser() - ) - # Attempt to obtain password. Short circuit evaluation at the first - # sign of success. - self.password = ( - self.password or - self._load_password_from_keyring() or - self._prompt_for_password() + def run(self): + msg = ( + "The upload command has been removed, use twine to upload " + + "instead (https://pypi.org/p/twine)" ) - def upload_file(self, command, pyversion, filename): - # Makes sure the repository URL is compliant - schema, netloc, url, params, query, fragments = \ - urlparse(self.repository) - if params or query or fragments: - raise AssertionError("Incompatible url %s" % self.repository) - - if schema not in ('http', 'https'): - raise AssertionError("unsupported schema " + schema) - - # Sign if requested - if self.sign: - gpg_args = ["gpg", "--detach-sign", "-a", filename] - if self.identity: - gpg_args[2:2] = ["--local-user", self.identity] - spawn(gpg_args, - dry_run=self.dry_run) - - # Fill in the data - send all the meta-data in case we need to - # register a new release - with open(filename, 'rb') as f: - content = f.read() - - meta = self.distribution.metadata - - data = { - # action - ':action': 'file_upload', - 'protocol_version': '1', - - # identify release - 'name': meta.get_name(), - 'version': meta.get_version(), - - # file content - 'content': (os.path.basename(filename), content), - 'filetype': command, - 'pyversion': pyversion, - 'md5_digest': hashlib.md5(content).hexdigest(), - - # additional meta-data - 'metadata_version': str(meta.get_metadata_version()), - 'summary': meta.get_description(), - 'home_page': meta.get_url(), - 'author': meta.get_contact(), - 'author_email': meta.get_contact_email(), - 'license': meta.get_licence(), - 'description': meta.get_long_description(), - 'keywords': meta.get_keywords(), - 'platform': meta.get_platforms(), - 'classifiers': meta.get_classifiers(), - 'download_url': meta.get_download_url(), - # PEP 314 - 'provides': meta.get_provides(), - 'requires': meta.get_requires(), - 'obsoletes': meta.get_obsoletes(), - } - - data['comment'] = '' - - if self.sign: - data['gpg_signature'] = (os.path.basename(filename) + ".asc", - open(filename+".asc", "rb").read()) - - # set up the authentication - user_pass = (self.username + ":" + self.password).encode('ascii') - # The exact encoding of the authentication string is debated. - # Anyway PyPI only accepts ascii for both username or password. - auth = "Basic " + standard_b64encode(user_pass).decode('ascii') - - # Build up the MIME payload for the POST data - boundary = '--------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - sep_boundary = b'\r\n--' + boundary.encode('ascii') - end_boundary = sep_boundary + b'--\r\n' - body = io.BytesIO() - for key, value in data.items(): - title = '\r\nContent-Disposition: form-data; name="%s"' % key - # handle multiple entries for the same name - if not isinstance(value, list): - value = [value] - for value in value: - if type(value) is tuple: - title += '; filename="%s"' % value[0] - value = value[1] - else: - value = str(value).encode('utf-8') - body.write(sep_boundary) - body.write(title.encode('utf-8')) - body.write(b"\r\n\r\n") - body.write(value) - body.write(end_boundary) - body = body.getvalue() - - msg = "Submitting %s to %s" % (filename, self.repository) - self.announce(msg, log.INFO) - - # build the Request - headers = { - 'Content-type': 'multipart/form-data; boundary=%s' % boundary, - 'Content-length': str(len(body)), - 'Authorization': auth, - } - - request = Request(self.repository, data=body, - headers=headers) - # send the data - try: - result = urlopen(request) - status = result.getcode() - reason = result.msg - except HTTPError as e: - status = e.code - reason = e.msg - except OSError as e: - self.announce(str(e), log.ERROR) - raise - - if status == 200: - self.announce('Server response (%s): %s' % (status, reason), - log.INFO) - if self.show_response: - text = getattr(self, '_read_pypi_response', - lambda x: None)(result) - if text is not None: - msg = '\n'.join(('-' * 75, text, '-' * 75)) - self.announce(msg, log.INFO) - else: - msg = 'Upload failed (%s): %s' % (status, reason) - self.announce(msg, log.ERROR) - raise DistutilsError(msg) - - def _load_password_from_keyring(self): - """ - Attempt to load password from keyring. Suppress Exceptions. - """ - try: - keyring = __import__('keyring') - return keyring.get_password(self.repository, self.username) - except Exception: - pass - - def _prompt_for_password(self): - """ - Prompt for a password on the tty. Suppress Exceptions. - """ - try: - return getpass.getpass() - except (Exception, KeyboardInterrupt): - pass + self.announce("ERROR: " + msg, log.ERROR) + raise RemovedCommandError(msg) diff --git a/setuptools/errors.py b/setuptools/errors.py new file mode 100644 index 0000000000..2701747f56 --- /dev/null +++ b/setuptools/errors.py @@ -0,0 +1,16 @@ +"""setuptools.errors + +Provides exceptions used by setuptools modules. +""" + +from distutils.errors import DistutilsError + + +class RemovedCommandError(DistutilsError, RuntimeError): + """Error used for commands that have been removed in setuptools. + + Since ``setuptools`` is built on ``distutils``, simply removing a command + from ``setuptools`` will make the behavior fall back to ``distutils``; this + error is raised if a command exists in ``distutils`` but has been actively + removed in ``setuptools``. + """ diff --git a/setuptools/tests/test_register.py b/setuptools/tests/test_register.py index 96114595db..986058067b 100644 --- a/setuptools/tests/test_register.py +++ b/setuptools/tests/test_register.py @@ -1,43 +1,22 @@ -import mock -from distutils import log - -import pytest - from setuptools.command.register import register from setuptools.dist import Distribution +from setuptools.errors import RemovedCommandError +try: + from unittest import mock +except ImportError: + import mock -class TestRegisterTest: - def test_warns_deprecation(self): - dist = Distribution() - - cmd = register(dist) - cmd.run_command = mock.Mock() - cmd.send_metadata = mock.Mock() - cmd.announce = mock.Mock() - - cmd.run() +import pytest - cmd.announce.assert_called_with( - "WARNING: Registering is deprecated, use twine to upload instead " - "(https://pypi.org/p/twine/)", - log.WARN - ) - def test_warns_deprecation_when_raising(self): +class TestRegister: + def test_register_exception(self): + """Ensure that the register command has been properly removed.""" dist = Distribution() + dist.dist_files = [(mock.Mock(), mock.Mock(), mock.Mock())] cmd = register(dist) - cmd.run_command = mock.Mock() - cmd.send_metadata = mock.Mock() - cmd.send_metadata.side_effect = Exception - cmd.announce = mock.Mock() - with pytest.raises(Exception): + with pytest.raises(RemovedCommandError): cmd.run() - - cmd.announce.assert_called_with( - "WARNING: Registering is deprecated, use twine to upload instead " - "(https://pypi.org/p/twine/)", - log.WARN - ) diff --git a/setuptools/tests/test_upload.py b/setuptools/tests/test_upload.py index 320c6959da..7586cb262d 100644 --- a/setuptools/tests/test_upload.py +++ b/setuptools/tests/test_upload.py @@ -1,213 +1,22 @@ -import mock -import os -import re - -from distutils import log -from distutils.errors import DistutilsError - -import pytest - from setuptools.command.upload import upload from setuptools.dist import Distribution -from setuptools.extern import six - - -def _parse_upload_body(body): - boundary = u'\r\n----------------GHSKFJDLGDS7543FJKLFHRE75642756743254' - entries = [] - name_re = re.compile(u'^Content-Disposition: form-data; name="([^\"]+)"') - - for entry in body.split(boundary): - pair = entry.split(u'\r\n\r\n') - if not len(pair) == 2: - continue - - key, value = map(six.text_type.strip, pair) - m = name_re.match(key) - if m is not None: - key = m.group(1) - - entries.append((key, value)) - - return entries - - -@pytest.fixture -def patched_upload(tmpdir): - class Fix: - def __init__(self, cmd, urlopen): - self.cmd = cmd - self.urlopen = urlopen - - def __iter__(self): - return iter((self.cmd, self.urlopen)) - - def get_uploaded_metadata(self): - request = self.urlopen.call_args_list[0][0][0] - body = request.data.decode('utf-8') - entries = dict(_parse_upload_body(body)) - - return entries +from setuptools.errors import RemovedCommandError - class ResponseMock(mock.Mock): - def getheader(self, name, default=None): - """Mocked getheader method for response object""" - return { - 'content-type': 'text/plain; charset=utf-8', - }.get(name.lower(), default) +try: + from unittest import mock +except ImportError: + import mock - with mock.patch('setuptools.command.upload.urlopen') as urlopen: - urlopen.return_value = ResponseMock() - urlopen.return_value.getcode.return_value = 200 - urlopen.return_value.read.return_value = b'' - - content = os.path.join(str(tmpdir), "content_data") - - with open(content, 'w') as f: - f.write("Some content") - - dist = Distribution() - dist.dist_files = [('sdist', '3.7.0', content)] - - cmd = upload(dist) - cmd.announce = mock.Mock() - cmd.username = 'user' - cmd.password = 'hunter2' - - yield Fix(cmd, urlopen) - - -class TestUploadTest: - def test_upload_metadata(self, patched_upload): - cmd, patch = patched_upload - - # Set the metadata version to 2.1 - cmd.distribution.metadata.metadata_version = '2.1' - - # Run the command - cmd.ensure_finalized() - cmd.run() - - # Make sure we did the upload - patch.assert_called_once() - - # Make sure the metadata version is correct in the headers - entries = patched_upload.get_uploaded_metadata() - assert entries['metadata_version'] == '2.1' - - def test_warns_deprecation(self): - dist = Distribution() - dist.dist_files = [(mock.Mock(), mock.Mock(), mock.Mock())] - - cmd = upload(dist) - cmd.upload_file = mock.Mock() - cmd.announce = mock.Mock() - - cmd.run() +import pytest - cmd.announce.assert_called_once_with( - "WARNING: Uploading via this command is deprecated, use twine to " - "upload instead (https://pypi.org/p/twine/)", - log.WARN - ) - def test_warns_deprecation_when_raising(self): +class TestUpload: + def test_upload_exception(self): + """Ensure that the register command has been properly removed.""" dist = Distribution() dist.dist_files = [(mock.Mock(), mock.Mock(), mock.Mock())] cmd = upload(dist) - cmd.upload_file = mock.Mock() - cmd.upload_file.side_effect = Exception - cmd.announce = mock.Mock() - - with pytest.raises(Exception): - cmd.run() - - cmd.announce.assert_called_once_with( - "WARNING: Uploading via this command is deprecated, use twine to " - "upload instead (https://pypi.org/p/twine/)", - log.WARN - ) - - @pytest.mark.parametrize('url', [ - 'https://example.com/a;parameter', # Has parameters - 'https://example.com/a?query', # Has query - 'https://example.com/a#fragment', # Has fragment - 'ftp://example.com', # Invalid scheme - - ]) - def test_upload_file_invalid_url(self, url, patched_upload): - patched_upload.urlopen.side_effect = Exception("Should not be reached") - - cmd = patched_upload.cmd - cmd.repository = url - - cmd.ensure_finalized() - with pytest.raises(AssertionError): - cmd.run() - - def test_upload_file_http_error(self, patched_upload): - patched_upload.urlopen.side_effect = six.moves.urllib.error.HTTPError( - 'https://example.com', - 404, - 'File not found', - None, - None - ) - - cmd = patched_upload.cmd - cmd.ensure_finalized() - with pytest.raises(DistutilsError): + with pytest.raises(RemovedCommandError): cmd.run() - - cmd.announce.assert_any_call( - 'Upload failed (404): File not found', - log.ERROR) - - def test_upload_file_os_error(self, patched_upload): - patched_upload.urlopen.side_effect = OSError("Invalid") - - cmd = patched_upload.cmd - cmd.ensure_finalized() - - with pytest.raises(OSError): - cmd.run() - - cmd.announce.assert_any_call('Invalid', log.ERROR) - - @mock.patch('setuptools.command.upload.spawn') - def test_upload_file_gpg(self, spawn, patched_upload): - cmd, urlopen = patched_upload - - cmd.sign = True - cmd.identity = "Alice" - cmd.dry_run = True - content_fname = cmd.distribution.dist_files[0][2] - signed_file = content_fname + '.asc' - - with open(signed_file, 'wb') as f: - f.write("signed-data".encode('utf-8')) - - cmd.ensure_finalized() - cmd.run() - - # Make sure that GPG was called - spawn.assert_called_once_with([ - "gpg", "--detach-sign", "--local-user", "Alice", "-a", - content_fname - ], dry_run=True) - - # Read the 'signed' data that was transmitted - entries = patched_upload.get_uploaded_metadata() - assert entries['gpg_signature'] == 'signed-data' - - def test_show_response_no_error(self, patched_upload): - # This test is just that show_response doesn't throw an error - # It is not really important what the printed response looks like - # in a deprecated command, but we don't want to introduce new - # errors when importing this function from distutils - - patched_upload.cmd.show_response = True - patched_upload.cmd.ensure_finalized() - patched_upload.cmd.run() From 14c82188dae748fb7a7dd126fb2a5553e4865c95 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Tue, 8 Oct 2019 12:18:31 +0200 Subject: [PATCH 46/49] test: drop pkg_resources tests dependency on easy_install --- .../data/my-test-package-source/setup.cfg | 0 .../data/my-test-package-source/setup.py | 6 ++ .../EGG-INFO/PKG-INFO | 10 +++ .../EGG-INFO/SOURCES.txt | 7 +++ .../EGG-INFO/dependency_links.txt | 1 + .../EGG-INFO/top_level.txt | 1 + .../EGG-INFO/zip-safe | 1 + .../my_test_package-1.0-py3.7.egg | Bin 0 -> 843 bytes .../tests/test_find_distributions.py | 58 ++++-------------- pytest.ini | 2 +- 10 files changed, 40 insertions(+), 46 deletions(-) create mode 100644 pkg_resources/tests/data/my-test-package-source/setup.cfg create mode 100644 pkg_resources/tests/data/my-test-package-source/setup.py create mode 100644 pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/PKG-INFO create mode 100644 pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/SOURCES.txt create mode 100644 pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/dependency_links.txt create mode 100644 pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/top_level.txt create mode 100644 pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/zip-safe create mode 100644 pkg_resources/tests/data/my-test-package_zipped-egg/my_test_package-1.0-py3.7.egg diff --git a/pkg_resources/tests/data/my-test-package-source/setup.cfg b/pkg_resources/tests/data/my-test-package-source/setup.cfg new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pkg_resources/tests/data/my-test-package-source/setup.py b/pkg_resources/tests/data/my-test-package-source/setup.py new file mode 100644 index 0000000000..fe80d28f46 --- /dev/null +++ b/pkg_resources/tests/data/my-test-package-source/setup.py @@ -0,0 +1,6 @@ +import setuptools +setuptools.setup( + name="my-test-package", + version="1.0", + zip_safe=True, +) diff --git a/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/PKG-INFO b/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/PKG-INFO new file mode 100644 index 0000000000..7328e3f7d1 --- /dev/null +++ b/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/PKG-INFO @@ -0,0 +1,10 @@ +Metadata-Version: 1.0 +Name: my-test-package +Version: 1.0 +Summary: UNKNOWN +Home-page: UNKNOWN +Author: UNKNOWN +Author-email: UNKNOWN +License: UNKNOWN +Description: UNKNOWN +Platform: UNKNOWN diff --git a/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/SOURCES.txt b/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/SOURCES.txt new file mode 100644 index 0000000000..3c4ee1676d --- /dev/null +++ b/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/SOURCES.txt @@ -0,0 +1,7 @@ +setup.cfg +setup.py +my_test_package.egg-info/PKG-INFO +my_test_package.egg-info/SOURCES.txt +my_test_package.egg-info/dependency_links.txt +my_test_package.egg-info/top_level.txt +my_test_package.egg-info/zip-safe \ No newline at end of file diff --git a/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/dependency_links.txt b/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/dependency_links.txt new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/top_level.txt b/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/top_level.txt new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/top_level.txt @@ -0,0 +1 @@ + diff --git a/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/zip-safe b/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/zip-safe new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/pkg_resources/tests/data/my-test-package_unpacked-egg/my_test_package-1.0-py3.7.egg/EGG-INFO/zip-safe @@ -0,0 +1 @@ + diff --git a/pkg_resources/tests/data/my-test-package_zipped-egg/my_test_package-1.0-py3.7.egg b/pkg_resources/tests/data/my-test-package_zipped-egg/my_test_package-1.0-py3.7.egg new file mode 100644 index 0000000000000000000000000000000000000000..5115b8957da8213d02b718649c8702ac88bf0e72 GIT binary patch literal 843 zcmWIWW@Zs#U|`^2n5Zq`&m4DbLIsew8;Avg*wx)#*VE6*?dE<9UVG zTUYDcne&^246YbI_~d=YcWcmzwO4dKb@eXldib34^YMS`6etukYxeAjkj$k5ub#hs zF8}WM(~0wEbA`;B*CkM-qkHm%zSHy9%buyFJyUS0HJTj!xohH&~mV7-!xlG;;_T+IdotQYo{ zdscNMxpHZVUROLaU%=<_fu5xwzcf|f-Fr0ogxjs30UtHBu5OoL>{fg&CcQwgsq5zY z{)cy0Y^(lyJ^9V@^PAE-|c literal 0 HcmV?d00001 diff --git a/pkg_resources/tests/test_find_distributions.py b/pkg_resources/tests/test_find_distributions.py index d735c5902f..f9594422f2 100644 --- a/pkg_resources/tests/test_find_distributions.py +++ b/pkg_resources/tests/test_find_distributions.py @@ -1,17 +1,9 @@ -import subprocess -import sys - +import py import pytest import pkg_resources -SETUP_TEMPLATE = """ -import setuptools -setuptools.setup( - name="my-test-package", - version="1.0", - zip_safe=True, -) -""".lstrip() + +TESTS_DATA_DIR = py.path.local(__file__).dirpath('data') class TestFindDistributions: @@ -21,46 +13,22 @@ def target_dir(self, tmpdir): target_dir = tmpdir.mkdir('target') # place a .egg named directory in the target that is not an egg: target_dir.mkdir('not.an.egg') - return str(target_dir) - - @pytest.fixture - def project_dir(self, tmpdir): - project_dir = tmpdir.mkdir('my-test-package') - (project_dir / "setup.py").write(SETUP_TEMPLATE) - return str(project_dir) + return target_dir def test_non_egg_dir_named_egg(self, target_dir): - dists = pkg_resources.find_distributions(target_dir) + dists = pkg_resources.find_distributions(str(target_dir)) assert not list(dists) - def test_standalone_egg_directory(self, project_dir, target_dir): - # install this distro as an unpacked egg: - args = [ - sys.executable, - '-c', 'from setuptools.command.easy_install import main; main()', - '-mNx', - '-d', target_dir, - '--always-unzip', - project_dir, - ] - subprocess.check_call(args) - dists = pkg_resources.find_distributions(target_dir) + def test_standalone_egg_directory(self, target_dir): + (TESTS_DATA_DIR / 'my-test-package_unpacked-egg').copy(target_dir) + dists = pkg_resources.find_distributions(str(target_dir)) assert [dist.project_name for dist in dists] == ['my-test-package'] - dists = pkg_resources.find_distributions(target_dir, only=True) + dists = pkg_resources.find_distributions(str(target_dir), only=True) assert not list(dists) - def test_zipped_egg(self, project_dir, target_dir): - # install this distro as an unpacked egg: - args = [ - sys.executable, - '-c', 'from setuptools.command.easy_install import main; main()', - '-mNx', - '-d', target_dir, - '--zip-ok', - project_dir, - ] - subprocess.check_call(args) - dists = pkg_resources.find_distributions(target_dir) + def test_zipped_egg(self, target_dir): + (TESTS_DATA_DIR / 'my-test-package_zipped-egg').copy(target_dir) + dists = pkg_resources.find_distributions(str(target_dir)) assert [dist.project_name for dist in dists] == ['my-test-package'] - dists = pkg_resources.find_distributions(target_dir, only=True) + dists = pkg_resources.find_distributions(str(target_dir), only=True) assert not list(dists) diff --git a/pytest.ini b/pytest.ini index 5886973bad..0bc1ec0114 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,6 @@ [pytest] addopts=--doctest-modules --doctest-glob=pkg_resources/api_tests.txt -r sxX -norecursedirs=dist build *.egg setuptools/extern pkg_resources/extern tools .* +norecursedirs=dist build *.egg setuptools/extern pkg_resources/extern pkg_resources/tests/data tools .* flake8-ignore = setuptools/site-patch.py F821 setuptools/py*compat.py F811 From fed59d837495208c13cec64b5394cdd2cc3cb6de Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Thu, 14 Nov 2019 19:09:20 +0100 Subject: [PATCH 47/49] tests: fix some pytest warnings under Python 2 --- setuptools/tests/test_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setuptools/tests/test_integration.py b/setuptools/tests/test_integration.py index 1c0b2b18bb..f1a27f8be8 100644 --- a/setuptools/tests/test_integration.py +++ b/setuptools/tests/test_integration.py @@ -64,7 +64,7 @@ def fin(): monkeypatch.setattr('site.USER_BASE', user_base.strpath) monkeypatch.setattr('site.USER_SITE', user_site.strpath) monkeypatch.setattr('sys.path', sys.path + [install_dir.strpath]) - monkeypatch.setenv('PYTHONPATH', os.path.pathsep.join(sys.path)) + monkeypatch.setenv(str('PYTHONPATH'), str(os.path.pathsep.join(sys.path))) # Set up the command for performing the installation. dist = Distribution() From 77fa2369892b7e45ede9cad18aaa3f0721c96cc3 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Fri, 15 Nov 2019 19:06:26 +0100 Subject: [PATCH 48/49] tweak workaround for #1644 Work around buggy pip detection code for "pip.exe install/update pip ...". --- tools/tox_pip.py | 11 ++++++++++- tox.ini | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tools/tox_pip.py b/tools/tox_pip.py index 1117f99653..5aeca80503 100644 --- a/tools/tox_pip.py +++ b/tools/tox_pip.py @@ -17,12 +17,21 @@ def pip(args): 'pip']) shutil.rmtree(glob(os.path.join(TOX_PIP_DIR, 'pip-*.dist-info'))[0]) # And use that version. + pypath = os.environ.get('PYTHONPATH') + pypath = pypath.split(os.pathsep) if pypath is not None else [] + pypath.insert(0, TOX_PIP_DIR) + os.environ['PYTHONPATH'] = os.pathsep.join(pypath) + # Disable PEP 517 support when using editable installs. for n, a in enumerate(args): if not a.startswith('-'): if a in 'install' and '-e' in args[n:]: args.insert(n + 1, '--no-use-pep517') break - subprocess.check_call([sys.executable, os.path.join(TOX_PIP_DIR, 'pip')] + args) + # Fix call for setuptools editable install. + for n, a in enumerate(args): + if a == '.': + args[n] = os.getcwd() + subprocess.check_call([sys.executable, '-m', 'pip'] + args, cwd=TOX_PIP_DIR) if __name__ == '__main__': diff --git a/tox.ini b/tox.ini index 8b34c235c9..5d439cb34b 100644 --- a/tox.ini +++ b/tox.ini @@ -14,7 +14,7 @@ envlist=python pip = python {toxinidir}/tools/tox_pip.py [testenv] -deps=-rtests/requirements.txt +deps=-r{toxinidir}/tests/requirements.txt install_command = {[helpers]pip} install {opts} {packages} list_dependencies_command = {[helpers]pip} freeze --all setenv=COVERAGE_FILE={toxworkdir}/.coverage.{envname} From a00798264cf4d55db28585cfc147050a4d579b52 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 16 Nov 2019 12:49:03 -0500 Subject: [PATCH 49/49] Trim excess whitespace --- docs/setuptools.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/setuptools.txt b/docs/setuptools.txt index 399a56d3ca..b7fdf4105a 100644 --- a/docs/setuptools.txt +++ b/docs/setuptools.txt @@ -1207,7 +1207,7 @@ the quoted part. Distributing a ``setuptools``-based project =========================================== -Detailed instructions to distribute a setuptools project can be found at +Detailed instructions to distribute a setuptools project can be found at `Packaging project tutorials`_. .. _Packaging project tutorials: https://packaging.python.org/tutorials/packaging-projects/#generating-distribution-archives @@ -1223,7 +1223,7 @@ setup.py is located:: This will generate distribution archives in the `dist` directory. -Before you upload the generated archives make sure you're registered on +Before you upload the generated archives make sure you're registered on https://test.pypi.org/account/register/. You will also need to verify your email to be able to upload any packages. You should install twine to be able to upload packages::