Permalink
Browse files

Issue #24915: Add Clang support to PGO builds and use the test suite

for profile data.

Thanks to Alecsandru Patrascu of Intel for the initial patch.
  • Loading branch information...
brettcannon committed Sep 18, 2015
1 parent 4b363e2 commit 7188a3efe07b9effdb760f3a96783f250214f0be
Showing with 185 additions and 7 deletions.
  1. +3 −0 .gitignore
  2. +3 −0 .hgignore
  3. +24 −7 Makefile.pre.in
  4. +1 −0 Misc/ACKS
  5. +3 −0 Misc/NEWS
  6. +28 −0 README
  7. +80 −0 configure
  8. +43 −0 configure.ac
@@ -11,6 +11,9 @@
*.rej
*.swp
*~
*.gc??
*.profclang?
*.profraw
.gdb_history
Doc/build/
Doc/venv/
@@ -50,6 +50,9 @@ libpython*.so*
*.pyd
*.cover
*~
*.gc??
*.profclang?
*.profraw
Lib/distutils/command/*.pdb
Lib/lib2to3/*.pickle
Lib/test/data/*
@@ -43,6 +43,11 @@ LDVERSION= @LDVERSION@
HGVERSION= @HGVERSION@
HGTAG= @HGTAG@
HGBRANCH= @HGBRANCH@
PGO_PROF_GEN_FLAG=@PGO_PROF_GEN_FLAG@
PGO_PROF_USE_FLAG=@PGO_PROF_USE_FLAG@
LLVM_PROF_MERGER=@LLVM_PROF_MERGER@
LLVM_PROF_FILE=@LLVM_PROF_FILE@
LLVM_PROF_ERR=@LLVM_PROF_ERR@
GNULD= @GNULD@
@@ -226,8 +231,7 @@ TCLTK_INCLUDES= @TCLTK_INCLUDES@
TCLTK_LIBS= @TCLTK_LIBS@
# The task to run while instrument when building the profile-opt target
PROFILE_TASK= $(srcdir)/Tools/pybench/pybench.py -n 2 --with-gc --with-syscheck
#PROFILE_TASK= $(srcdir)/Lib/test/regrtest.py
PROFILE_TASK=-m test.regrtest >/dev/null 2>&1
# report files for gcov / lcov coverage report
COVERAGE_INFO= $(abs_builddir)/coverage.info
@@ -477,27 +481,38 @@ LIBRARY_OBJS= \
all: build_all
build_all: $(BUILDPYTHON) oldsharedmods sharedmods gdbhooks Programs/_testembed python-config
# Compile a binary with gcc profile guided optimization.
# Compile a binary with profile guided optimization.
profile-opt:
@if [ $(LLVM_PROF_ERR) == yes ]; then \
echo "Error: Cannot perform PGO build because llvm-profdata was not found in PATH" ;\
echo "Please add it to PATH and run ./configure again" ;\
exit 1;\
fi
@echo "Building with support for profile generation:"
$(MAKE) clean
$(MAKE) profile-removal
$(MAKE) build_all_generate_profile
@echo "Running benchmark to generate profile data:"
$(MAKE) profile-removal
@echo "Running code to generate profile data (this can take a while):"
$(MAKE) run_profile_task
$(MAKE) build_all_merge_profile
@echo "Rebuilding with profile guided optimizations:"
$(MAKE) clean
$(MAKE) build_all_use_profile
$(MAKE) profile-removal
build_all_generate_profile:
$(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-generate" LDFLAGS="-fprofile-generate" LIBS="$(LIBS) -lgcov"
$(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_GEN_FLAG)" LDFLAGS="$(LDFLAGS) $(PGO_PROF_GEN_FLAG)" LIBS="$(LIBS)"
run_profile_task:
: # FIXME: can't run for a cross build
$(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK)
$(LLVM_PROF_FILE) $(RUNSHARED) ./$(BUILDPYTHON) $(PROFILE_TASK) || true
build_all_merge_profile:
$(LLVM_PROF_MERGER)
build_all_use_profile:
$(MAKE) all CFLAGS_NODIST="$(CFLAGS) -fprofile-use -fprofile-correction"
$(MAKE) all CFLAGS_NODIST="$(CFLAGS) $(PGO_PROF_USE_FLAG)"
# Compile and run with gcov
.PHONY=coverage coverage-lcov coverage-report
@@ -1568,9 +1583,11 @@ clean: pycremoval
-rm -f pybuilddir.txt
-rm -f Lib/lib2to3/*Grammar*.pickle
-rm -f Programs/_testembed Programs/_freeze_importlib
-rm -rf build
profile-removal:
find . -name '*.gc??' -exec rm -f {} ';'
find . -name '*.profclang?' -exec rm -f {} ';'
rm -f $(COVERAGE_INFO)
rm -rf $(COVERAGE_REPORT)
@@ -1079,6 +1079,7 @@ Heikki Partanen
Harri Pasanen
Gaël Pasgrimaud
Ashish Nitin Patil
Alecsandru Patrascu
Randy Pausch
Samuele Pedroni
Justin Peel
@@ -115,6 +115,9 @@ Tests
Build
-----
- Issue #24915: Add LLVM support for PGO builds and use the test suite to
generate the profile data. Initiial patch by Alecsandru Patrascu of Intel.
- Issue #24910: Windows MSIs now have unique display names.
- Issue #24986: It is now possible to build Python on Windows without errors
28 README
@@ -46,6 +46,34 @@ For example:
(This will fail if you *also* built at the top-level directory.
You should do a "make clean" at the toplevel first.)
If you need an optimized version of Python, you type "make profile-opt" in the
top level directory. This will rebuild the interpreter executable using Profile
Guided Optimization (PGO). For more details, see the section bellow.
Profile Guided Optimization
---------------------------
PGO takes advantage of recent versions of the GCC or Clang compilers.
If ran, the "profile-opt" rule will do several steps.
First, the entire Python directory is cleaned of temporary files that
may resulted in a previous compilation.
Then, an instrumented version of the interpreter is built, using suitable
compiler flags for each flavour. Note that this is just an intermediary
step and the binary resulted after this step is not good for real life
workloads, as it has profiling instructions embedded inside.
After this instrumented version of the interpreter is built, the Makefile
will automatically run a training workload. This is necessary in order to
profile the interpreter execution. Note also that any output, both stdout
and stderr, that may appear at this step is supressed.
Finally, the last step is to rebuild the interpreter, using the information
collected in the previous one. The end result will be a the Python binary
that is optimized and suitable for distribution or production installation.
What's New
----------
@@ -667,6 +667,12 @@ UNIVERSAL_ARCH_FLAGS
CFLAGS_NODIST
BASECFLAGS
OPT
LLVM_PROF_FOUND
LLVM_PROF_ERR
LLVM_PROF_FILE
LLVM_PROF_MERGER
PGO_PROF_USE_FLAG
PGO_PROF_GEN_FLAG
ABIFLAGS
LN
MKDIR_P
@@ -6431,6 +6437,80 @@ $as_echo "no" >&6; }
fi
# Enable PGO flags.
# Extract the first word of "llvm-profdata", so it can be a program name with args.
set dummy llvm-profdata; ac_word=$2
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5
$as_echo_n "checking for $ac_word... " >&6; }
if ${ac_cv_prog_LLVM_PROF_FOUND+:} false; then :
$as_echo_n "(cached) " >&6
else
if test -n "$LLVM_PROF_FOUND"; then
ac_cv_prog_LLVM_PROF_FOUND="$LLVM_PROF_FOUND" # Let the user override the test.
else
as_save_IFS=$IFS; IFS=$PATH_SEPARATOR
for as_dir in $PATH
do
IFS=$as_save_IFS
test -z "$as_dir" && as_dir=.
for ac_exec_ext in '' $ac_executable_extensions; do
if as_fn_executable_p "$as_dir/$ac_word$ac_exec_ext"; then
ac_cv_prog_LLVM_PROF_FOUND="found"
$as_echo "$as_me:${as_lineno-$LINENO}: found $as_dir/$ac_word$ac_exec_ext" >&5
break 2
fi
done
done
IFS=$as_save_IFS
test -z "$ac_cv_prog_LLVM_PROF_FOUND" && ac_cv_prog_LLVM_PROF_FOUND="not-found"
fi
fi
LLVM_PROF_FOUND=$ac_cv_prog_LLVM_PROF_FOUND
if test -n "$LLVM_PROF_FOUND"; then
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $LLVM_PROF_FOUND" >&5
$as_echo "$LLVM_PROF_FOUND" >&6; }
else
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
$as_echo "no" >&6; }
fi
LLVM_PROF_ERR=no
case $CC in
*clang*)
# Any changes made here should be reflected in the GCC+Darwin case below
PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
if test $LLVM_PROF_FOUND = not-found
then
LLVM_PROF_ERR=yes
fi
;;
*gcc*)
case $ac_sys_system in
Darwin*)
PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
if test $LLVM_PROF_FOUND = not-found
then
LLVM_PROF_ERR=yes
fi
;;
*)
PGO_PROF_GEN_FLAG="-fprofile-generate"
PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
LLVM_PROF_MERGER="true"
LLVM_PROF_FILE=""
;;
esac
;;
esac
# XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be
# merged with this chunk of code?
@@ -1218,6 +1218,49 @@ else AC_MSG_RESULT(no); Py_DEBUG='false'
fi],
[AC_MSG_RESULT(no)])
# Enable PGO flags.
AC_SUBST(PGO_PROF_GEN_FLAG)
AC_SUBST(PGO_PROF_USE_FLAG)
AC_SUBST(LLVM_PROF_MERGER)
AC_SUBST(LLVM_PROF_FILE)
AC_SUBST(LLVM_PROF_ERR)
AC_SUBST(LLVM_PROF_FOUND)
AC_CHECK_PROG(LLVM_PROF_FOUND, llvm-profdata, found, not-found)
LLVM_PROF_ERR=no
case $CC in
*clang*)
# Any changes made here should be reflected in the GCC+Darwin case below
PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
if test $LLVM_PROF_FOUND = not-found
then
LLVM_PROF_ERR=yes
fi
;;
*gcc*)
case $ac_sys_system in
Darwin*)
PGO_PROF_GEN_FLAG="-fprofile-instr-generate"
PGO_PROF_USE_FLAG="-fprofile-instr-use=code.profclangd"
LLVM_PROF_MERGER="llvm-profdata merge -output=code.profclangd *.profclangr"
LLVM_PROF_FILE="LLVM_PROFILE_FILE=\"code-%p.profclangr\""
if test $LLVM_PROF_FOUND = not-found
then
LLVM_PROF_ERR=yes
fi
;;
*)
PGO_PROF_GEN_FLAG="-fprofile-generate"
PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction"
LLVM_PROF_MERGER="true"
LLVM_PROF_FILE=""
;;
esac
;;
esac
# XXX Shouldn't the code above that fiddles with BASECFLAGS and OPT be
# merged with this chunk of code?

0 comments on commit 7188a3e

Please sign in to comment.