From 96bf295e884e3eaf6627b3c9e8d1cab448b23618 Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Wed, 1 Nov 2017 13:47:05 -0700 Subject: [PATCH 1/3] bpo-28643: Record profile-opt build progress with stamp files The profile-opt makefile target is expensive to build. Since the makefile does not contain complete dependency information for this target, much extra work can get done if the build is interrupted and re-started. Even running "make" a second time will result in a huge amount of redundant work. As a minimal fix (rather than removing recursive "make" and adding a proper dependency graph), split the profile-opt target into three parts: - build with profile generation enabled (profile-gen-stamp) - run task to generate profile information (profile-run-stamp) - build optimized Python using above information (profile-opt) Use the "stamp" files profile-gen-stamp and profile-run-stamp to record completion of the first two steps. The first will be cleaned by the "clobber" target, the second by "profile-removal". Other minor changes: - remove the "build_all_use_profile" target. I don't expect callers of the makefile to use this target so that should be safe. - remove "$(MAKE) clean" at the start of the profile-gen-stamp target. I assume that a profiled build normally starts from a clean source or build tree and so cleaning should not be required. Having the clean there causes an interruption of the profile-gen result in everthing getting built from scratch. - remove execution of "profile-removal" at end of "profile-opt". I don't see any reason to not to keep the profile information, given the cost to generate it. Removing the stamp files will force a re-build. --- Makefile.pre.in | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 6dacb872e0c243..40272579ab99a0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -442,25 +442,30 @@ DTRACE_DEPS = \ all: @DEF_MAKE_ALL_RULE@ build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config -# Compile a binary with profile guided optimization. -profile-opt: +# Compile with profile generation +profile-gen-stamp: @if [ $(LLVM_PROF_ERR) = yes ]; then \ echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\ echo "Please add it to PATH and run ./configure again" ;\ exit 1;\ fi @echo "Building with support for profile generation:" - $(MAKE) clean $(MAKE) profile-removal $(MAKE) build_all_generate_profile - $(MAKE) profile-removal + touch $@ + +# Run task to generate profile information +profile-run-stamp: profile-gen-stamp @echo "Running code to generate profile data (this can take a while):" + $(MAKE) profile-removal $(MAKE) run_profile_task $(MAKE) build_all_merge_profile - @echo "Rebuilding with profile guided optimizations:" $(MAKE) clean - $(MAKE) build_all_use_profile - $(MAKE) profile-removal + # This is an expensive target to build and it does not have proper + # makefile dependancy information. So, we create a "stamp" file + # to record that it has been preformed and does not have to get + # done again. + touch $@ build_all_generate_profile: $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)" @@ -472,7 +477,9 @@ run_profile_task: build_all_merge_profile: $(LLVM_PROF_MERGER) -build_all_use_profile: +# Compile a binary with profile guided optimization. +profile-opt: profile-run-stamp + @echo "Rebuilding with profile guided optimizations:" $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_USE_FLAG)" LDFLAGS="$(LDFLAGS)" # Compile and run with gcov @@ -1628,6 +1635,7 @@ profile-removal: find . -name '*.dyn' -exec rm -f {} ';' rm -f $(COVERAGE_INFO) rm -rf $(COVERAGE_REPORT) + rm -f profile-run-stamp clobber: clean profile-removal -rm -f $(BUILDPYTHON) $(PGEN) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \ @@ -1636,6 +1644,7 @@ clobber: clean profile-removal -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) -rm -f python-config.py python-config + -rm -f profile-gen-stamp # Make things extra clean, before making a distribution: # remove all generated files, even Makefile[.pre] From 3b6a65735dbbeb02217c72da1c540b9a5aa8757f Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Wed, 1 Nov 2017 14:19:11 -0700 Subject: [PATCH 2/3] Add blurb. Add extra comments to makefile regarding stamp files. --- Makefile.pre.in | 6 ++++++ .../next/Build/2017-11-01-14-16-27.bpo-28643.9iPKJy.rst | 1 + 2 files changed, 7 insertions(+) create mode 100644 Misc/NEWS.d/next/Build/2017-11-01-14-16-27.bpo-28643.9iPKJy.rst diff --git a/Makefile.pre.in b/Makefile.pre.in index 40272579ab99a0..5f5dffe78f804e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -443,6 +443,9 @@ all: @DEF_MAKE_ALL_RULE@ build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config # Compile with profile generation +# +# Run "make clean" and remove stamp file to force re-build of the profile +# generation binary. profile-gen-stamp: @if [ $(LLVM_PROF_ERR) = yes ]; then \ echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\ @@ -455,6 +458,9 @@ profile-gen-stamp: touch $@ # Run task to generate profile information +# +# Run "make clean" and remove stamp file to force re-running of profile +# generation task. profile-run-stamp: profile-gen-stamp @echo "Running code to generate profile data (this can take a while):" $(MAKE) profile-removal diff --git a/Misc/NEWS.d/next/Build/2017-11-01-14-16-27.bpo-28643.9iPKJy.rst b/Misc/NEWS.d/next/Build/2017-11-01-14-16-27.bpo-28643.9iPKJy.rst new file mode 100644 index 00000000000000..87f6d5825d2886 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2017-11-01-14-16-27.bpo-28643.9iPKJy.rst @@ -0,0 +1 @@ +Record profile-opt build progress with stamp files. From cd7ed5f9c844add412d32881aada1b766b17f58c Mon Sep 17 00:00:00 2001 From: Neil Schemenauer Date: Thu, 2 Nov 2017 10:10:46 -0700 Subject: [PATCH 3/3] Rework stamp files for profile-opt build. Add a profile-clean-stamp target. We must make sure that Python is built from a clean tree with -fprofile-generate before running the profile task. --- Makefile.pre.in | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 5f5dffe78f804e..e5d65bde26658e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -442,35 +442,36 @@ DTRACE_DEPS = \ all: @DEF_MAKE_ALL_RULE@ build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config -# Compile with profile generation -# -# Run "make clean" and remove stamp file to force re-build of the profile -# generation binary. -profile-gen-stamp: +# Profile generation build must start from a clean tree. +profile-clean-stamp: + $(MAKE) clean profile-removal + touch $@ + +# Compile with profile generation enabled. +profile-gen-stamp: profile-clean-stamp @if [ $(LLVM_PROF_ERR) = yes ]; then \ echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\ echo "Please add it to PATH and run ./configure again" ;\ exit 1;\ fi @echo "Building with support for profile generation:" - $(MAKE) profile-removal $(MAKE) build_all_generate_profile touch $@ -# Run task to generate profile information -# -# Run "make clean" and remove stamp file to force re-running of profile -# generation task. -profile-run-stamp: profile-gen-stamp +# Run task with profile generation build to create profile information. +profile-run-stamp: @echo "Running code to generate profile data (this can take a while):" - $(MAKE) profile-removal + # First, we need to create a clean build with profile generation + # enabled. + $(MAKE) profile-gen-stamp + # Next, run the profile task to generate the profile information. $(MAKE) run_profile_task $(MAKE) build_all_merge_profile + # Remove profile generation binary since we are done with it. $(MAKE) clean # This is an expensive target to build and it does not have proper # makefile dependancy information. So, we create a "stamp" file - # to record that it has been preformed and does not have to get - # done again. + # to record its completion and avoid re-running it. touch $@ build_all_generate_profile: @@ -483,9 +484,11 @@ run_profile_task: build_all_merge_profile: $(LLVM_PROF_MERGER) -# Compile a binary with profile guided optimization. +# Compile Python binary with profile guided optimization. +# To force re-running of the profile task, remove the profile-run-stamp file. profile-opt: profile-run-stamp @echo "Rebuilding with profile guided optimizations:" + -rm -f profile-clean-stamp $(MAKE) @DEF_MAKE_RULE@ CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_USE_FLAG)" LDFLAGS="$(LDFLAGS)" # Compile and run with gcov @@ -1634,6 +1637,7 @@ clean: pycremoval -rm -f Programs/_testembed Programs/_freeze_importlib -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h + -rm -f profile-gen-stamp profile-removal: find . -name '*.gc??' -exec rm -f {} ';' @@ -1650,7 +1654,7 @@ clobber: clean profile-removal -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) -rm -f python-config.py python-config - -rm -f profile-gen-stamp + -rm -f profile-gen-stamp profile-clean-stamp # Make things extra clean, before making a distribution: # remove all generated files, even Makefile[.pre]