From 62388c4b144fc6df55f036ba540d106a8382cf95 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 18 Feb 2017 13:23:09 +0900 Subject: [PATCH 1/9] Add PYTHONFRAMEWORK to pyconfig.h --- configure.ac | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configure.ac b/configure.ac index 18b940ab329172..815769f534bedd 100644 --- a/configure.ac +++ b/configure.ac @@ -355,6 +355,8 @@ AC_SUBST(FRAMEWORKPYTHONW) AC_SUBST(FRAMEWORKUNIXTOOLSPREFIX) AC_SUBST(FRAMEWORKINSTALLAPPSPREFIX) +AC_DEFINE_UNQUOTED(PYTHONFRAMEWORK, "${PYTHONFRAMEWORK}", [framework name]) + ##AC_ARG_WITH(dyld, ## AS_HELP_STRING([--with-dyld], ## [Use (OpenStep|Rhapsody) dynamic linker])) From 2e12d4217047aba989af6a71dd6b6f780d40eb42 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Sat, 18 Feb 2017 13:23:35 +0900 Subject: [PATCH 2/9] Add sys._framework --- Python/sysmodule.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 424a88f7086b3e..84673e3fed9737 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1965,6 +1965,7 @@ _PySys_BeginInit(void) SET_SYS_FROM_STRING("_git", Py_BuildValue("(szz)", "CPython", _Py_gitidentifier(), _Py_gitversion())); + SET_SYS_FROM_STRING("_framework", PyUnicode_FromString(PYTHONFRAMEWORK)); SET_SYS_FROM_STRING("api_version", PyLong_FromLong(PYTHON_API_VERSION)); SET_SYS_FROM_STRING("copyright", From ff0d05c3a018b53afd5b13255f31ff4422b56a53 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 16 Feb 2017 17:45:38 +0900 Subject: [PATCH 3/9] optimize site.py Skip importing sysconfig when possible. Median +- std dev: [default] 15.8 ms +- 0.0 ms -> [patched] 14.7 ms +- 0.0 ms: 1.07x faster (-7%) --- Lib/site.py | 58 +++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index 87979383584066..500c59b0a97102 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -124,7 +124,7 @@ def removeduppaths(): # if they only differ in case); turn relative paths into absolute # paths. dir, dircase = makepath(dir) - if not dircase in known_paths: + if dircase not in known_paths: L.append(dir) known_paths.add(dircase) sys.path[:] = L @@ -234,6 +234,27 @@ def check_enableusersite(): return True + +def _getuserbase(): + # Stripped version of sysconfig._getuserbase() + env_base = os.environ.get("PYTHONUSERBASE", None) + if env_base: + return env_base + + def joinuser(*args): + return os.path.expanduser(os.path.join(*args)) + + if os.name == "nt": + base = os.environ.get("APPDATA") or "~" + return joinuser(base, "Python") + + if sys.platform == "darwin" and sys._framework: + return joinuser("~", "Library", sys._framework, + "%d.%d" % sys.version_info[:2]) + + return joinuser("~", ".local") + + def getuserbase(): """Returns the `user base` directory path. @@ -242,12 +263,24 @@ def getuserbase(): it. """ global USER_BASE - if USER_BASE is not None: - return USER_BASE - from sysconfig import get_config_var - USER_BASE = get_config_var('userbase') + if USER_BASE is None: + USER_BASE = _getuserbase() return USER_BASE + +def _get_path(userbase): + # stripped version of sysconfig.get_path('purelib', os.name + '_user') + version = sys.version_info[:2] + + if os.name == 'nt': + return f'{userbase}/Python{version[0]}{version[1]}/site-packages' + + if sys.platform == 'darwin' and sys._framework: + return f'{userbase}/lib/python/site-packages' + + return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages' + + def getusersitepackages(): """Returns the user-specific site-packages directory path. @@ -255,20 +288,11 @@ def getusersitepackages(): function will also set it. """ global USER_SITE - user_base = getuserbase() # this will also set USER_BASE - - if USER_SITE is not None: - return USER_SITE - - from sysconfig import get_path + userbase = getuserbase() # this will also set USER_BASE - if sys.platform == 'darwin': - from sysconfig import get_config_var - if get_config_var('PYTHONFRAMEWORK'): - USER_SITE = get_path('purelib', 'osx_framework_user') - return USER_SITE + if USER_SITE is None: + USER_SITE = _get_path(userbase) - USER_SITE = get_path('purelib', '%s_user' % os.name) return USER_SITE def addusersitepackages(known_paths): From a1976a7faa9e677ab8ff903e0ad8569645c98b05 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 28 Jun 2017 19:30:55 +0900 Subject: [PATCH 4/9] run autoreconf --- configure | 20 +++++++++++++++++++- pyconfig.h.in | 3 +++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/configure b/configure index ec42e08f8961c1..ff8152d7d19312 100755 --- a/configure +++ b/configure @@ -783,6 +783,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -896,6 +897,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1148,6 +1150,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1285,7 +1296,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1438,6 +1449,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -3231,6 +3243,12 @@ fi + +cat >>confdefs.h <<_ACEOF +#define PYTHONFRAMEWORK "${PYTHONFRAMEWORK}" +_ACEOF + + ##AC_ARG_WITH(dyld, ## AS_HELP_STRING([--with-dyld], ## [Use (OpenStep|Rhapsody) dynamic linker])) diff --git a/pyconfig.h.in b/pyconfig.h.in index fa2792b18ad419..f7c50eadcafc12 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1247,6 +1247,9 @@ /* Define as the preferred size in bits of long digits */ #undef PYLONG_BITS_IN_DIGIT +/* framework name */ +#undef PYTHONFRAMEWORK + /* Define if you want to coerce the C locale to a UTF-8 based locale */ #undef PY_COERCE_C_LOCALE From 5aabcec1e633feac7ac2a171001f87e483852ada Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 28 Jun 2017 19:40:07 +0900 Subject: [PATCH 5/9] Add note comment for duplicated code. --- Lib/sysconfig.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index ed0a34d662f189..92c930c9aef556 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -51,6 +51,7 @@ 'scripts': '{base}/Scripts', 'data': '{base}', }, + # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { 'stdlib': '{userbase}/Python{py_version_nodot}', 'platstdlib': '{userbase}/Python{py_version_nodot}', @@ -177,6 +178,8 @@ def _get_default_scheme(): return os.name +# NOTE: site.py has stripped copy of this function. +# Update it too when modify this function. def _getuserbase(): env_base = os.environ.get("PYTHONUSERBASE", None) From 6f7830335b32093c9022ecca7283c8fe20908ba2 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 28 Jun 2017 21:25:21 +0900 Subject: [PATCH 6/9] remove sysconfig import on darwin platform --- Lib/site.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index 500c59b0a97102..0835485b48a012 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -334,15 +334,11 @@ def getsitepackages(prefixes=None): else: sitepackages.append(prefix) sitepackages.append(os.path.join(prefix, "lib", "site-packages")) - if sys.platform == "darwin": - # for framework builds *only* we add the standard Apple - # locations. - from sysconfig import get_config_var - framework = get_config_var("PYTHONFRAMEWORK") - if framework: - sitepackages.append( - os.path.join("/Library", framework, - '%d.%d' % sys.version_info[:2], "site-packages")) + # for framework builds *only* we add the standard Apple locations. + if sys.platform == "darwin" and sys._framework: + sitepackages.append( + os.path.join("/Library", framework, + '%d.%d' % sys.version_info[:2], "site-packages")) return sitepackages def addsitepackages(known_paths, prefixes=None): From 33c420ac292a365ed06ea5105cc4298bccf9d4d9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Wed, 28 Jun 2017 22:02:10 +0900 Subject: [PATCH 7/9] More elaborate comments about copied functions. --- Lib/site.py | 34 ++++++++++++++++++++-------------- Lib/sysconfig.py | 27 +++++++++------------------ 2 files changed, 29 insertions(+), 32 deletions(-) diff --git a/Lib/site.py b/Lib/site.py index 0835485b48a012..fcf7dde41131ac 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -235,8 +235,14 @@ def check_enableusersite(): return True +# NOTE: sysconfig and it's dependencies are relatively large but site module +# needs very limited part of them. +# To speedup startup time, we have copy of them. +# +# See https://bugs.python.org/issue29585 + +# Copy of sysconfig._getuserbase() def _getuserbase(): - # Stripped version of sysconfig._getuserbase() env_base = os.environ.get("PYTHONUSERBASE", None) if env_base: return env_base @@ -255,6 +261,19 @@ def joinuser(*args): return joinuser("~", ".local") +# Same to sysconfig.get_path('purelib', os.name+'_user') +def _get_path(userbase): + version = sys.version_info + + if os.name == 'nt': + return f'{userbase}/Python{version[0]}{version[1]}/site-packages' + + if sys.platform == 'darwin' and sys._framework: + return f'{userbase}/lib/python/site-packages' + + return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages' + + def getuserbase(): """Returns the `user base` directory path. @@ -268,19 +287,6 @@ def getuserbase(): return USER_BASE -def _get_path(userbase): - # stripped version of sysconfig.get_path('purelib', os.name + '_user') - version = sys.version_info[:2] - - if os.name == 'nt': - return f'{userbase}/Python{version[0]}{version[1]}/site-packages' - - if sys.platform == 'darwin' and sys._framework: - return f'{userbase}/lib/python/site-packages' - - return f'{userbase}/lib/python{version[0]}.{version[1]}/site-packages' - - def getusersitepackages(): """Returns the user-specific site-packages directory path. diff --git a/Lib/sysconfig.py b/Lib/sysconfig.py index 92c930c9aef556..e6618b1d5182ca 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig.py @@ -178,34 +178,25 @@ def _get_default_scheme(): return os.name -# NOTE: site.py has stripped copy of this function. -# Update it too when modify this function. +# NOTE: site.py has copy of this function. +# Sync it when modify this function. def _getuserbase(): env_base = os.environ.get("PYTHONUSERBASE", None) + if env_base: + return env_base def joinuser(*args): return os.path.expanduser(os.path.join(*args)) if os.name == "nt": base = os.environ.get("APPDATA") or "~" - if env_base: - return env_base - else: - return joinuser(base, "Python") + return joinuser(base, "Python") - if sys.platform == "darwin": - framework = get_config_var("PYTHONFRAMEWORK") - if framework: - if env_base: - return env_base - else: - return joinuser("~", "Library", framework, "%d.%d" % - sys.version_info[:2]) + if sys.platform == "darwin" and sys._framework: + return joinuser("~", "Library", sys._framework, + "%d.%d" % sys.version_info[:2]) - if env_base: - return env_base - else: - return joinuser("~", ".local") + return joinuser("~", ".local") def _parse_makefile(filename, vars=None): From bdabfd0bd8694616c9f136182431f75043bc23f6 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 29 Jun 2017 00:14:07 +0900 Subject: [PATCH 8/9] add test for copied two functions --- Lib/test/test_site.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 150162279e6df6..bf7be4ec1c6a20 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -180,6 +180,13 @@ def test_addsitedir(self): finally: pth_file.cleanup() + def test_getuserbase(self): + self.assertEqual(site._getuserbase(), sysconfig._getuserbase()) + + def test_get_path(self): + self.assertEqual(site._get_path(site._getuserbase()), + sysconfig.get_path('purelib', os.name + '_user')) + @unittest.skipUnless(site.ENABLE_USER_SITE, "requires access to PEP 370 " "user-site (site.ENABLE_USER_SITE)") def test_s_option(self): From 484634915f93c6eb59d6fd549adcd1e377faa3c9 Mon Sep 17 00:00:00 2001 From: INADA Naoki Date: Thu, 29 Jun 2017 00:17:48 +0900 Subject: [PATCH 9/9] Add NEWS entry --- .../next/Library/2017-06-29-00-17-38.bpo-29585.x2V0my.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2017-06-29-00-17-38.bpo-29585.x2V0my.rst diff --git a/Misc/NEWS.d/next/Library/2017-06-29-00-17-38.bpo-29585.x2V0my.rst b/Misc/NEWS.d/next/Library/2017-06-29-00-17-38.bpo-29585.x2V0my.rst new file mode 100644 index 00000000000000..c841f1b2250efc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-06-29-00-17-38.bpo-29585.x2V0my.rst @@ -0,0 +1,2 @@ +Avoid importing ``sysconfig`` from ``site`` to improve startup speed. Python +startup is about 5% faster on Linux and 30% faster on macOS.