From 77878eb22a9a9784647af0a234adc9f1358c0d46 Mon Sep 17 00:00:00 2001 From: Max Maischein Date: Mon, 28 Dec 2020 13:00:49 +0100 Subject: [PATCH] WIP: Run `makedepend` in parallel by using `make` This moves the per-file loop body of `makedepend` into a separate file named `makedepend_file` and then uses `make` to launch the `makedepend_file` processes for each target potentially in parallel. This reduces the time for time sh ./makedepend MAKE=make cflags from 5 seconds to 2 seconds with MAKEFLAGS=-j8 --- .gitignore | 1 + MANIFEST | 1 + Makefile.SH | 6 +- makedepend.SH | 121 ++++--------------------------- makedepend_file.SH | 177 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 197 insertions(+), 109 deletions(-) create mode 100644 makedepend_file.SH diff --git a/.gitignore b/.gitignore index 4fcfe623c7f9..caaa58c5129b 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ bug*.pl /config.sh /makeaperl /makedepend +/makedepend_file /makedir /makefile /myconfig diff --git a/MANIFEST b/MANIFEST index 1d334a514cb6..d1ba6768b942 100644 --- a/MANIFEST +++ b/MANIFEST @@ -4984,6 +4984,7 @@ make_ext.pl Used by Makefile to execute extension Makefiles make_patchnum.pl Script to generate git_version.h and lib/Config_git.pl files for all OS'es makedef.pl Create symbol export lists for linking makedepend.SH Precursor to makedepend +makedepend_file.SH Precursor to makedepend_file Makefile.micro microperl Makefile Makefile.SH A script that generates Makefile malloc.c A version of malloc you might not want diff --git a/Makefile.SH b/Makefile.SH index ba8d9bd19d3c..56ef5e266d30 100755 --- a/Makefile.SH +++ b/Makefile.SH @@ -1377,6 +1377,7 @@ veryclean: _verycleaner _mopup _clobber _mopup: rm -f *$(OBJ_EXT) *$(LIB_EXT) all perlmain.c opmini.c perlmini.c generate_uudmap$(EXE_EXT) $(generated_headers) -rmdir .depending + -rm *.depends -@test -f extra.pods && rm -f `cat extra.pods` -@test -f vms/README_vms.pod && rm -f vms/README_vms.pod -rm -f perl.exp ext.libs $(generated_pods) uni.data opmini.o perlmini.o pod/roffitall @@ -1500,7 +1501,7 @@ cscope.out cscope: $(c) $(h) # The README below ensures that the dependency list is never empty and # that when MAKEDEPEND is empty $(FIRSTMAKEFILE) doesn't need rebuilding. -MAKEDEPEND = Makefile makedepend +MAKEDEPEND = Makefile makedepend_file makedepend $(FIRSTMAKEFILE): README $(MAKEDEPEND) $(MAKE) depend MAKEDEPEND= @@ -1521,6 +1522,9 @@ $spitshell >>$Makefile <<'!NO!SUBS!' depend: makedepend $(DTRACE_H) $(generated_headers) sh ./makedepend MAKE="$(MAKE)" cflags +%.c.depends: %.c + sh ./makedepend_file $< $@ cflags + .PHONY: test check test_prep test_prep_nodll test_prep_pre \ test_prep_reonly test_tty test-tty test_notty test-notty \ test_harness test_harness_notty minitest test-reonly _test diff --git a/makedepend.SH b/makedepend.SH index 600288f3de04..5cf5a38c7789 100755 --- a/makedepend.SH +++ b/makedepend.SH @@ -36,9 +36,9 @@ fi mkdir .depending -# This script should be called with +# This script should be called with # sh ./makedepend MAKE=$(MAKE) -case "$1" in +case "$1" in MAKE=*) eval $1; shift ;; esac @@ -112,114 +112,17 @@ $test -d UU || mkdir UU $MAKE clist || ($echo "Searching for .c files..."; \ $echo *.c | $tr ' ' $trnl | $egrep -v '\*' >.clist) -for file in `$cat .clist`; do -# for file in `cat /dev/null`; do - case "$osname" in - uwin) uwinfix="-e s,\\\\\\\\,/,g -e s,\\([a-zA-Z]\\):/,/\\1/,g" ;; - os2) uwinfix="-e s,\\\\\\\\,/,g" ;; - cygwin) uwinfix="-e s,\\\\\\\\,/,g" ;; - posix-bc) uwinfix="-e s/\\*POSIX(\\(.*\\))/\\1/" ;; - vos) uwinfix="-e s/\#/\\\#/" ;; - *) uwinfix="" ;; - esac - case "$file" in - *.c) filebase=`basename $file .c` ;; - *.y) filebase=`basename $file .y` ;; - esac - case "$file" in - */*) finc="-I`echo $file | sed 's#/[^/]*$##'`" ;; - *) finc= ;; - esac - $echo "Finding dependencies for $filebase$_o" - # Below, we strip out all but preprocessor directives. - # We have to take care of situations like - # #if defined(FOO) BAR /* comment line 1 - # more comment lines */ - # If we just delete text starting from the '/*' to the end of line, we will - # screw up cases like - # #if defined(FOO) /* comment */ \ - # && defined(BAR) /* comment */ \ - # && defined(BAZ) /* comment */ \ - # etc. - # Also, in lines like - # #defined FOO(a,b) a/**/b - # the comment may be important and so needs to be retained. - # This code processes the single-line comments first; it assumes there is - # at most one straightforward comment per continued preprocessor line, - # replacing each non-empty comment (and its surrounding white space) by a - # single space. (sed only has a greedy '*' quantifier, so this doesn't - # work right if there are multiple comments per line, and strings can look - # like comments to it; both are unlikely in a preprocessor statement.) Any - # continuation line is joined, and the process repeated on the enlarged - # line as long as there are continuations. At the end, if there are any - # comments remaining, they are either completely empty or are like the - # first situation. The latter are just deleted by first deleting to the - # end of line (including preceding white space) things that start with '/*' - # and the next char isn't a '*'; then things that start with '/**', but the - # next char isn't a '/'. (Subsequent lines of the comment are irrelevant - # and get dropped.) At the end, we unjoin very long lines to avoid - # preprocessor limitations - ( $echo "#line 2 \"$file\""; \ - $sed -n <$file \ - -e "/^${filebase}_init(/q" \ - -e ': tstcont' \ - -e '/^[ ]*#/s|[ ]*/\*..*\*/[ ]*| |' \ - -e '/\\$/{' \ - -e 'N' \ - -e 'b tstcont' \ - -e '}' \ - -e 's/\\\n//g' \ - -e '/^#line/d' \ - -e '/^[ ]*#/{' \ - -e 's|[ ]*/\*[^*].*$||' \ - -e 's|[ ]*/\*\*[^/].*$||' \ - -e 's/.\{255\}/&\\\n/g' \ - -e p \ - -e '}' ) >UU/$file.c - # We're not sure why this was there; the #endif is extraneous on modern z/OS - #if [ "$osname" = os390 -a "$file" = perly.c ]; then - # $echo '#endif' >>UU/$file.c - #fi +clist=`$cat .clist | $sed -e 's,$,.depends,'` - if [ "$osname" = os390 ]; then - $cppstdin $finc -I. $cppflags $cppminus /d' \ - -e '/^#.*"-"/d' \ - -e '/^#.*git_version\.h/d' \ - -e 's#\.[0-9][0-9]*\.c#'"$file.c#" \ - -e 's/^[ ]*#[ ]*line/#/' \ - -e '/^# *[0-9][0-9]* *[".\/]/!d' \ - -e 's/^.*"\(.*\)".*$/'$filebase'\$(OBJ_EXT): \1/' \ - -e 's/^# *[0-9][0-9]* \(.*\)$/'$filebase'\$(OBJ_EXT): \1/' \ - -e 's|: \./|: |' \ - -e 's|\.c\.c|.c|' $uwinfix | \ - $uniq | $sort | $uniq >> .deptmp - else - $cppstdin $finc -I. $cppflags $cppminus .cout 2>.cerr - $sed \ - -e '1d' \ - -e '/^#.*/d' \ - -e '/^#.*/d' \ - -e '/^#.*/d' \ - -e '/^#.*/d' \ - -e '/^#.*/d' \ - -e '/^#.*"-"/d' \ - -e '/^#.*"\/.*\/"/d' \ - -e '/: file path prefix .* never used$/d' \ - -e '/^#.*git_version\.h/d' \ - -e 's#\.[0-9][0-9]*\.c#'"$file.c#" \ - -e 's/^[ ]*#[ ]*line/#/' \ - -e '/^# *[0-9][0-9]* *[".\/]/!d' \ - -e 's/^.*"\(.*\)".*$/'$filebase'\$(OBJ_EXT): \1/' \ - -e 's/^# *[0-9][0-9]* \(.*\)$/'$filebase'\$(OBJ_EXT): \1/' \ - -e 's|: \./|: |' \ - -e 's|\.c\.c|.c|' $uwinfix .cout .cerr| \ - $uniq | $sort | $uniq >> .deptmp - fi - echo "$filebase\$(OBJ_EXT): $@" >> .deptmp -done +# Now, create a Makefile from .clist and run that in parallel +# Makefiles creating more Makefiles +# A Makefile exists to beget more Makefiles. Perl is +# just a vehicle. +rm -f $clist +$MAKE $clist +$cat $clist >.deptmp +rm -f $clist $sed <$mf >$mf.new -e '1,/^# AUTOMATICALLY/!d' @@ -235,6 +138,8 @@ if $test -s .deptmp; then -e 'h; s/mini\(perlmain\)/\1/p; g' \ .deptmp >>$mf.new else + $echo "Should not get here" + exit 1 $MAKE hlist || ($echo "Searching for .h files..."; \ $echo *.h | $tr ' ' $trnl | $egrep -v '\*' >.hlist) $echo "You don't seem to have a proper C preprocessor. Using grep instead." diff --git a/makedepend_file.SH b/makedepend_file.SH new file mode 100644 index 000000000000..51f5191fe938 --- /dev/null +++ b/makedepend_file.SH @@ -0,0 +1,177 @@ +#! /bin/sh +case $PERL_CONFIG_SH in +'') + if test -f config.sh; then TOP=.; + elif test -f ../config.sh; then TOP=..; + elif test -f ../../config.sh; then TOP=../..; + elif test -f ../../../config.sh; then TOP=../../..; + elif test -f ../../../../config.sh; then TOP=../../../..; + else + echo "Can't find config.sh."; exit 1 + fi + . $TOP/config.sh + ;; +esac +: This forces SH files to create target in same directory as SH file. +: This is so that make depend always knows where to find SH derivatives. +case "$0" in +*/*) cd `expr X$0 : 'X\(.*\)/'` ;; +esac + +echo "Extracting makedepend_file (with variable substitutions)" +rm -f makedepend_file +$spitshell >makedepend_file <>makedepend_file <<'!NO!SUBS!' + +file=$1 +shift +outfile=$1 +shift + +case $PERL_CONFIG_SH in +'') + if test -f config.sh; then TOP=.; + elif test -f ../config.sh; then TOP=..; + elif test -f ../../config.sh; then TOP=../..; + elif test -f ../../../config.sh; then TOP=../../..; + elif test -f ../../../../config.sh; then TOP=../../../..; + else + echo "Can't find config.sh."; exit 1 + fi + . $TOP/config.sh + ;; +esac + +# Avoid localized gcc messages +case "$ccname" in + gcc) LC_ALL=C ; export LC_ALL ;; +esac + +# We need .. when we are in the x2p directory if we are using the +# cppstdin wrapper script. +# Put .. and . first so that we pick up the present cppstdin, not +# an older one lying about in /usr/local/bin. +PATH=".$path_sep..$path_sep$PATH" +export PATH + +case "$osname" in +amigaos) cat=/bin/cat ;; # must be absolute +esac + + case "$osname" in + uwin) uwinfix="-e s,\\\\\\\\,/,g -e s,\\([a-zA-Z]\\):/,/\\1/,g" ;; + os2) uwinfix="-e s,\\\\\\\\,/,g" ;; + cygwin) uwinfix="-e s,\\\\\\\\,/,g" ;; + posix-bc) uwinfix="-e s/\\*POSIX(\\(.*\\))/\\1/" ;; + vos) uwinfix="-e s/\#/\\\#/" ;; + *) uwinfix="" ;; + esac + case "$file" in + *.c) filebase=`basename $file .c` ;; + *.y) filebase=`basename $file .y` ;; + esac + case "$file" in + */*) finc="-I`echo $file | sed 's#/[^/]*$##'`" ;; + *) finc= ;; + esac + $echo "Finding dependencies for $filebase$_o" + # Below, we strip out all but preprocessor directives. + # We have to take care of situations like + # #if defined(FOO) BAR /* comment line 1 + # more comment lines */ + # If we just delete text starting from the '/*' to the end of line, we will + # screw up cases like + # #if defined(FOO) /* comment */ \ + # && defined(BAR) /* comment */ \ + # && defined(BAZ) /* comment */ \ + # etc. + # Also, in lines like + # #defined FOO(a,b) a/**/b + # the comment may be important and so needs to be retained. + # This code processes the single-line comments first; it assumes there is + # at most one straightforward comment per continued preprocessor line, + # replacing each non-empty comment (and its surrounding white space) by a + # single space. (sed only has a greedy '*' quantifier, so this doesn't + # work right if there are multiple comments per line, and strings can look + # like comments to it; both are unlikely in a preprocessor statement.) Any + # continuation line is joined, and the process repeated on the enlarged + # line as long as there are continuations. At the end, if there are any + # comments remaining, they are either completely empty or are like the + # first situation. The latter are just deleted by first deleting to the + # end of line (including preceding white space) things that start with '/*' + # and the next char isn't a '*'; then things that start with '/**', but the + # next char isn't a '/'. (Subsequent lines of the comment are irrelevant + # and get dropped.) At the end, we unjoin very long lines to avoid + # preprocessor limitations + ( $echo "#line 2 \"$file\""; \ + $sed -n <$file \ + -e "/^${filebase}_init(/q" \ + -e ': tstcont' \ + -e '/^[ ]*#/s|[ ]*/\*..*\*/[ ]*| |' \ + -e '/\\$/{' \ + -e 'N' \ + -e 'b tstcont' \ + -e '}' \ + -e 's/\\\n//g' \ + -e '/^#line/d' \ + -e '/^[ ]*#/{' \ + -e 's|[ ]*/\*[^*].*$||' \ + -e 's|[ ]*/\*\*[^/].*$||' \ + -e 's/.\{255\}/&\\\n/g' \ + -e p \ + -e '}' | $grep -v ":" ) >UU/$file.c + + # We're not sure why this was there; the #endif is extraneous on modern z/OS + #if [ "$osname" = os390 -a "$file" = perly.c ]; then + # $echo '#endif' >>UU/$file.c + #fi + + if [ "$osname" = os390 ]; then + $cppstdin $finc -I. $cppflags $cppminus /d' \ + -e '/^#.*"-"/d' \ + -e '/^#.*git_version\.h/d' \ + -e 's#\.[0-9][0-9]*\.c#'"$file.c#" \ + -e 's/^[ ]*#[ ]*line/#/' \ + -e '/^# *[0-9][0-9]* *[".\/]/!d' \ + -e 's/^.*"\(.*\)".*$/'$filebase'\$(OBJ_EXT): \1/' \ + -e 's/^# *[0-9][0-9]* \(.*\)$/'$filebase'\$(OBJ_EXT): \1/' \ + -e 's|: \./|: |' \ + -e 's|\.c\.c|.c|' $uwinfix | \ + $uniq | $sort | $uniq >>$outfile + else + $cppstdin $finc -I. $cppflags $cppminus $$.cout 2>$$.cerr + $sed \ + -e '1d' \ + -e '/^#.*/d' \ + -e '/^#.*/d' \ + -e '/^#.*/d' \ + -e '/^#.*/d' \ + -e '/^#.*/d' \ + -e '/^#.*"-"/d' \ + -e '/^#.*"\/.*\/"/d' \ + -e '/: file path prefix .* never used$/d' \ + -e '/^#.*git_version\.h/d' \ + -e 's#\.[0-9][0-9]*\.c#'"$file.c#" \ + -e 's/^[ ]*#[ ]*line/#/' \ + -e '/^# *[0-9][0-9]* *[".\/]/!d' \ + -e 's/^.*"\(.*\)".*$/'$filebase'\$(OBJ_EXT): \1/' \ + -e 's/^# *[0-9][0-9]* \(.*\)$/'$filebase'\$(OBJ_EXT): \1/' \ + -e 's|: \./|: |' \ + -e 's|\.c\.c|.c|' $uwinfix $$.cout $$.cerr| \ + $uniq | $sort | $uniq >>$outfile + rm $$.cout + rm $$.cerr + fi + $echo "$filebase\$(OBJ_EXT): $@" >>$outfile + +!NO!SUBS! +$eunicefix makedepend_file +chmod +x makedepend_file