Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Commit

Permalink
python: slow startup fix for all pyc files
Browse files Browse the repository at this point in the history
(unrelated: Also gets rid of static lib.)

NOTE: Rebuild python-* packages (meson, ...) to get the fix.
  • Loading branch information
dylanaraps committed Aug 10, 2021
1 parent 42f4937 commit 2ef9124
Show file tree
Hide file tree
Showing 5 changed files with 75 additions and 13 deletions.
16 changes: 3 additions & 13 deletions extra/python/build
@@ -1,14 +1,5 @@
#!/bin/sh -e

# Without this environment variable being set to some arbitrary value,
# python's module loader uses timestamps for cache invalidation. Something
# goes wrong here and all pyc files are seen as stale.
#
# By setting this value, python uses checksums for cache invalidation which
# (on my machine) drops startup time from 3 seconds to 0.060s. Further
# investigation is needed.
export SOURCE_DATE_EPOCH=1628593994

# Remove util-linux dependency among other things.
cat >> Modules/Setup <<EOF
*disabled*
Expand All @@ -20,7 +11,9 @@ EOF
export CFLAGS="$CFLAGS -fno-semantic-interposition"
export LDFLAGS="$LDFLAGS -fno-semantic-interposition"

patch -p1 < python3-always-pip.patch
for patch in *.patch; do
patch -p1 < "$patch"
done

./configure \
--prefix=/usr \
Expand All @@ -36,9 +29,6 @@ make install
ln -s python3 "$1/usr/bin/python"
ln -s pip3 "$1/usr/bin/pip"

# Make static library writable.
chmod u+w "$1/usr/lib/libpython"*

# Let's make some kind of effort to reduce the overall
# size of Python by removing a bunch of rarely used and
# otherwise useless components.
Expand Down
2 changes: 2 additions & 0 deletions extra/python/checksums
@@ -1,2 +1,4 @@
397920af33efc5b97f2e0b57e91923512ef89fc5b3c1d21dbfc8c4828ce0108a
1e3dd58370fa3f5c9454f72479565e5282d16defdeec43c88e9a4dd233e00c1b
18ec738f531ff250c96b74f80bdba58433a39302ab69895593cc2fa2bdf4d0aa
e39e9562cf5b864500ef358960545787761fc4271829d22b19c893d1ce0e4546
52 changes: 52 additions & 0 deletions extra/python/patches/python3-no-static.patch
@@ -0,0 +1,52 @@
diff --git a/Makefile.pre.in b/Makefile.pre.in
index 2b68571..41871c9 100644
--- a/Makefile.pre.in
+++ b/Makefile.pre.in
@@ -589,7 +589,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --srcdir $(srcdir)

# Build the interpreter
-$(BUILDPYTHON): Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) $(EXPORTSYMS)
+$(BUILDPYTHON): Programs/python.o $(LDLIBRARY) $(PY3LIBRARY) $(EXPORTSYMS)
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)

platform: $(BUILDPYTHON) pybuilddir.txt
@@ -638,11 +638,6 @@ sharedmods: $(BUILDPYTHON) pybuilddir.txt Modules/_math.o
$(PYTHON_FOR_BUILD) $(srcdir)/setup.py $$quiet build


-# Build static library
-$(LIBRARY): $(LIBRARY_OBJS)
- -rm -f $@
- $(AR) $(ARFLAGS) $@ $(LIBRARY_OBJS)
-
libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS)
if test $(INSTSONAME) != $(LDLIBRARY); then \
$(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \
@@ -724,7 +719,7 @@ Makefile Modules/config.c: Makefile.pre \
@echo "The Makefile was updated, you may need to re-run make."


-Programs/_testembed: Programs/_testembed.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) $(EXPORTSYMS)
+Programs/_testembed: Programs/_testembed.o $(LDLIBRARY) $(PY3LIBRARY) $(EXPORTSYMS)
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)

############################################################################
@@ -1652,17 +1647,6 @@ libainstall: @DEF_MAKE_RULE@ python-config
else true; \
fi; \
done
- @if test -d $(LIBRARY); then :; else \
- if test "$(PYTHONFRAMEWORKDIR)" = no-framework; then \
- if test "$(SHLIB_SUFFIX)" = .dll; then \
- $(INSTALL_DATA) $(LDLIBRARY) $(DESTDIR)$(LIBPL) ; \
- else \
- $(INSTALL_DATA) $(LIBRARY) $(DESTDIR)$(LIBPL)/$(LIBRARY) ; \
- fi; \
- else \
- echo Skip install of $(LIBRARY) - use make frameworkinstall; \
- fi; \
- fi
$(INSTALL_DATA) Modules/config.c $(DESTDIR)$(LIBPL)/config.c
$(INSTALL_DATA) Programs/python.o $(DESTDIR)$(LIBPL)/python.o
$(INSTALL_DATA) $(srcdir)/Modules/config.c.in $(DESTDIR)$(LIBPL)/config.c.in
16 changes: 16 additions & 0 deletions extra/python/patches/python3-pyc-hash.patch
@@ -0,0 +1,16 @@
diff --git a/Lib/py_compile.py b/Lib/py_compile.py
index a81f493..e1328ed 100644
--- a/Lib/py_compile.py
+++ b/Lib/py_compile.py
@@ -70,10 +70,7 @@ class PycInvalidationMode(enum.Enum):


def _get_default_invalidation_mode():
- if os.environ.get('SOURCE_DATE_EPOCH'):
- return PycInvalidationMode.CHECKED_HASH
- else:
- return PycInvalidationMode.TIMESTAMP
+ return PycInvalidationMode.CHECKED_HASH


def compile(file, cfile=None, dfile=None, doraise=False, optimize=-1,
2 changes: 2 additions & 0 deletions extra/python/sources
@@ -1,2 +1,4 @@
https://www.python.org/ftp/python/VERSION/Python-VERSION.tar.xz
patches/python3-always-pip.patch
patches/python3-no-static.patch
patches/python3-pyc-hash.patch

0 comments on commit 2ef9124

Please sign in to comment.