diff --git a/utah/scripts/john_driver/evaluate_program.pl b/utah/scripts/john_driver/evaluate_program.pl new file mode 100755 index 000000000..be62499be --- /dev/null +++ b/utah/scripts/john_driver/evaluate_program.pl @@ -0,0 +1,1099 @@ +#!/usr/bin/perl + +use strict; +use warnings; +use Benchmark; +use English; +use LockFile::Simple; + +# TODO: check for program crash (vs. timeout) + +# TODO: factor out version search code; automatically check +# if it valgrinds clean; when we do a version search, save +# the temp dir + +# TODO: we hold the lock for too long, put it into compile_llvm? + +# abstract over compilation and evaluation strategies + +# properly propagate run_program failures upwards + +# piece together byte-level information from run_program + +# distinguish between programs that compile and those that run inside the timeout + +#################################################################### + +my $SKIP = 0; + +my $NOTEFILE_PREFIX="logged_"; + +my $xxtra = "-DCSMITH_MINIMAL"; + +#my $notmp = "-DUSE_MATH_MACROS_NOTMP"; +my $notmp = ""; + +#my $CCOMP_BF = "-fbitfields"; +my $CCOMP_BF = "-fbitfields"; + +# I don't know the reason, but on our mini mac, current-gcc complains about "can't find -lgcc" +# So remove the comment below if we do tests on maya +#my $CCOMP_EXTRA_LDIR = "-L/Users/Shared/compilers/lib/gcc/x86_64-apple-darwin10.2.0/4.5.0"; +my $CCOMP_EXTRA_LDIR = ""; + +my $COMPILER_TIMEOUT = 600; + +my $COMPILER_TIMEOUT_RES = 137; + +# see if it helps to wrap volatile accesses in function calls? +my $TRY_WRAP = 0; + +my $NO_VOLCHECK = 0; +my $USE_VOLCHECK = 1; +my $USE_PINTOOL = 2; + +my $CHECK_VOLATILE = $NO_VOLCHECK; + +if (defined($ENV{"PIN_CMD"})) { + $CHECK_VOLATILE = $USE_PINTOOL; +} + +my $VALGRIND_ON_COMPILER = 0; + +my $RUN_PROGRAM = 1; + +my $DO_TRIAGE = 0; + +my $VERSION_SEARCH = 0; + +my $BAIL_ON_ZERO_WRITES = 0; + +#################################################################### + +my $CSMITH_PATH=$ENV{"CSMITH_PATH"}; +die "oops: CSMITH_PATH environment variable needs to be set" + if (!defined($CSMITH_PATH)); + +my $VOLATILE_PATH=$ENV{"VOLATILE_PATH"}; +die "oops: VOLATILE_PATH environment variable needs to be set" + if (!defined($VOLATILE_PATH)); + +my $LOCKFN = "/var/tmp/version_search_lockfile"; + +#################################################################### + +my @gcc_opts2 = ( + "-O0", + "-O1", + "-O2", + "-Os", + "-O3", + ); + +my $DRAGONEGG = "-fplugin=/home/regehr/z/compiler-source/dragonegg/dragonegg.so"; + +my @dragonegg_opts = ( + "-O0 $DRAGONEGG", + "-O1 $DRAGONEGG", + "-O2 $DRAGONEGG", + "-Os $DRAGONEGG", + "-O3 $DRAGONEGG", + ); + +my @gcc_opts3 = ( + "-O0", + "-O1 -fomit-frame-pointer -fwhole-program", + "-O2", + "-Os", + "-Ofast -funroll-loops -fomit-frame-pointer -fwhole-program", + ); + +my @gcc_opts4 = ( + "-O0", + "-Os", + "-Os -fno-asynchronous-unwind-tables -fno-common -fno-delete-null-pointer-checks -fno-dwarf2-cfi-asm -fno-omit-frame-pointer -fno-optimize-sibling-calls -fno-strict-aliasing -fno-strict-overflow -fstack-protector -fstack-protector-all -ftest-coverage -g -m32 -march=i486 -mno-3dnow -mno-mmx -mno-sse -mno-sse2 -mpreferred-stack-boundary=2 -msoft-float", + ); + +my @gcc_opts5 = ( + "-O0", + "-O1", + "-O2 -march=core2 -mtune=core2 -fno-stack-protector -fomit-frame-pointer", + "-Os", + "-Os -fconserve-stack -fno-asynchronous-unwind-tables -fno-common -fno-delete-null-pointer-checks -fno-dwarf2-cfi-asm -fno-omit-frame-pointer -fno-optimize-sibling-calls -fno-strict-aliasing -fno-strict-overflow -fprofile-arcs -fstack-protector -ftest-coverage -funit-at-a-time -g -m64 -maccumulate-outgoing-args -mno-3dnow -mno-mmx -mno-red-zone -mno-sse -mno-sse2 -mtune=generic -pg", + "-O3 -fwhole-program -combine", + ); + +my @clang_opts = ( + "-O0 ", + "-O1 ", + "-O2 -march=core2 -mtune=core2 -fomit-frame-pointer -fno-stack-protector ", + "-Os ", + "-O3 ", + ); + +my @clangwrapv_opts = ( + "-O0 -fwrapv", + "-O1 -fwrapv", + "-O2 -fwrapv -march=core2 -mtune=core2 -fomit-frame-pointer -fno-stack-protector", + "-Os -fwrapv", + "-O3 -fwrapv", + ); + +my @gcc_686_opts = ( + "-O0 -march=i686", + "-O1 -march=i686", + "-O2 -march=i686", + "-Os -march=i686", + "-O3 -march=i686", + ); + +my @llvm_opts_sse3 = ( + "-O0 -msse3", + "-O1 -msse3", + "-O2 -msse3", + "-Os -msse3", + "-O3 -msse3", + ); + +my @armcc_opts = ( + "-O0", + "-O1", + "-O2 -Ospace", + "-O2 -Otime", + "-O3 -Ospace", + "-O3 -Otime", + ); + +my @dcc_opts = ( + "-tARMES:windiss -DSTANDALONE", + "-tARMES:windiss -DSTANDALONE -O", + "-tARMES:windiss -DSTANDALONE -XO", + ); + +my @gcc_opts_noO0 = ("-O1", + "-Os", + "-O2", + "-O3"); + +my @gcc_opts = ("-O0", + "-O1", + "-Os", + "-O2", + "-O3"); + +my @Oonly = ("-O"); + +my @Osonly = ("-Os"); + +my @none = ("-DTCC"); + +my @O0sonly = ("-O0", "-Os"); + +my @O0only = ("-O0"); + +my @suncc_opts = ( + "", + "-xO1", + "-xO2", + "-xO3", + "-xO4", + "-xO5", + "-fast", + ); + +my @icc_opts = ( + "-O0", + "-O1", + "-Os", + "-O2", + "-fast -ipo", + ); + +my @gcc_with_ccomp_opts = ( + "-m32 -O0 -DUSE_MATH_MACROS_NOTMP -D__COMPCERT__ $CCOMP_EXTRA_LDIR", + "-m32 -O1 -DUSE_MATH_MACROS_NOTMP -D__COMPCERT__ $CCOMP_EXTRA_LDIR", + "-m32 -O2 -DUSE_MATH_MACROS_NOTMP -D__COMPCERT__ $CCOMP_EXTRA_LDIR", + "-m32 -Os -DUSE_MATH_MACROS_NOTMP -D__COMPCERT__ $CCOMP_EXTRA_LDIR", + "-m32 -O3 -DUSE_MATH_MACROS_NOTMP -D__COMPCERT__ $CCOMP_EXTRA_LDIR", + ); + +my @ccomp_opts = ( + "-D__COMPCERT__ -DUSE_MATH_MACROS_NOTMP -fstruct-passing -fstruct-assign $CCOMP_BF", +); + +my @avrdeputy = ("avr", + "deputy", + "my-deputy-avr.pl", + \@Osonly); + +my @clang_Oonly = ("ia32", + "clang", + "clang", + \@Oonly); + +my @lance = ("ia32", + "clang", + "run-lance", + \@Osonly); + +my @tendra = ("ia32", + "tendracc", + "tendracc", + \@gcc_opts2); + +my @llvmgcc = ("ia32", + "llvm", + "llvm-gcc", + \@gcc_opts2); + +my @cparser = ("ia32", + "cparser", + "cparser-driver", + \@Osonly); + +my @tcc = ("ia32", + "tcc", + "tcc", + \@none); + +my @dcc = ("windiss", + "dcc", + "dcc", + \@dcc_opts); + +my @armcc = ("arm", + "armcc", + "armcc", + \@armcc_opts); + +my @avrgcc = ("avr", + "gcc", + "avr-gcc", + \@gcc_opts2); + +my @avrgcc34 = ("avr", + "gcc", + "avr-gcc-3.4", + \@gcc_opts2); + +my @mspgcc = ("msp", + "gcc", + "msp430-gcc", + \@gcc_opts2); + +my @avrgcc42_O0sonly = ("avr", + "gcc", + "avr-gcc-4.2", + \@O0sonly); + +my @avrgcc42 = ("avr", + "gcc", + "avr-gcc-4.2", + \@gcc_opts2); + +my @deputy = ("ia32", + "deputy", + "/home/regehr/tmp/deputy/bin/deputy", + \@Osonly); + +my @nesc = ("ia32", + "nesc", + "nescc", + \@Osonly); + +my @gcc = ("ia32", + "gcc", + "gcc", + \@gcc_opts2); + +my @gcc460 = ("ia32", + "gcc", + "gcc-460", + \@gcc_opts2); + +my @gcc450 = ("ia32", + "gcc", + "gcc-450", + \@gcc_opts2); + +my @gcc440 = ("ia32", + "gcc", + "gcc-440", + \@gcc_opts2); + +my @gcc430 = ("ia32", + "gcc", + "gcc-430", + \@gcc_opts2); + +my @gcc420 = ("ia32", + "gcc", + "gcc-420", + \@gcc_opts2); + +my @gcc410 = ("ia32", + "gcc", + "gcc-410", + \@gcc_opts2); + +my @gcc400 = ("ia32", + "gcc", + "gcc-400", + \@gcc_opts2); + +my @gcc340 = ("ia32", + "gcc", + "gcc-340", + \@gcc_opts2); + +my @gcc330 = ("ia32", + "gcc", + "gcc-330", + \@gcc_opts2); + +my @gcc320 = ("ia32", + "gcc", + "gcc-320", + \@gcc_opts2); + +my @gcc310 = ("ia32", + "gcc", + "gcc-310", + \@gcc_opts2); + +my @gcc300 = ("ia32", + "gcc", + "gcc-300", + \@gcc_opts2); + +my @clang = ("ia32", + "clang", + "clang", + \@gcc_opts2); + +my @clangpp = ("ia32", + "clang", + "clang++", + \@gcc_opts2); + +my @gcccurrent = ("ia32", + "gcc", + "current-gcc", + \@gcc_opts3); + +my @gppcurrent = ("ia32", + "gcc", + "current-g++", + \@gcc_opts3); + +my @dragonegg = ("ia32", + "gcc", + "gcc-4.5", + \@dragonegg_opts); + +my @gcccurrent2 = ("ia32", + "gcc", + "current-gcc", + \@gcc_opts4); + +my @gcccurrent_Oonly = ("ia32", + "gcc", + "current-gcc", + \@Oonly); + +my @gcccurrent_O0only = ("ia32", + "gcc", + "current-gcc", + \@O0only); + +my @suncc = ("ia32", + "suncc", + "suncc", + \@suncc_opts); + +my @llvmgcc19 = ("ia32", + "llvm", + "llvm-gcc-19", + \@gcc_opts2); + +my @llvmgcc20 = ("ia32", + "llvm", + "llvm-gcc-20", + \@gcc_opts2); + +my @llvmgcc21 = ("ia32", + "llvm", + "llvm-gcc-21", + \@gcc_opts2); + +my @llvmgcc22 = ("ia32", + "llvm", + "llvm-gcc-22", + \@gcc_opts2); + +my @llvmgcc23 = ("ia32", + "llvm", + "llvm-gcc-23", + \@gcc_opts2); + +my @llvmgcc24 = ("ia32", + "llvm", + "llvm-gcc-24", + \@gcc_opts2); + +my @llvmgcc25 = ("ia32", + "llvm", + "llvm-gcc-25", + \@gcc_opts2); + +my @llvmgcc26 = ("ia32", + "llvm", + "llvm-gcc-26", + \@gcc_opts2); + +my @clang26 = ("ia32", + "llvm", + "clang-26", + \@gcc_opts2); + +my @llvmgcc27 = ("ia32", + "llvm", + "llvm-gcc-27", + \@gcc_opts2); + +my @clang27 = ("ia32", + "llvm", + "clang-27", + \@gcc_opts2); + +my @llvmgcc28 = ("ia32", + "llvm", + "llvm-gcc-28", + \@gcc_opts2); + +my @clang28 = ("ia32", + "llvm", + "clang-28", + \@gcc_opts2); + +my @llvmgcc29 = ("ia32", + "llvm", + "llvm-gcc-29", + \@gcc_opts2); + +my @clang29 = ("ia32", + "llvm", + "clang-29", + \@gcc_opts2); + +my @llvmgcc22O0only = ("ia32", + "llvm", + "llvm-gcc-2.2", + \@O0only); + +my @llvmgcccurrent = ("ia32", + "llvm", + "llvm-gcc-current", + \@gcc_opts2); + +my @llvmgcc_Oonly = ("ia32", + "llvm", + "llvm-gcc", + \@Oonly); + +my @icc = ("ia32", + "icc", + "icc", + \@icc_opts); + +my @icc_O0only = ("ia32", + "icc", + "icc", + \@O0only); + +my @gcc_with_ccomp = ( + "ia32", + "gcc", + "current-gcc", + \@gcc_with_ccomp_opts); + +my @clang_with_ccomp = ( + "ia32", + "gcc", + "clang", + \@gcc_with_ccomp_opts); + +my @ccomp = ( + "ia32", + "ccomp", + "ccomp", + \@ccomp_opts); + +my @llvms = ( + \@llvmgcc20, + \@llvmgcc21, + \@llvmgcc22, + \@llvmgcc23, + \@llvmgcc24, + ); + +my @compilers_to_test = ( + + #\@gcc, + + \@clang, + #\@gcccurrent, + + #\@clangpp, + #\@gppcurrent, + + #\@gcc400, + + #\@tcc, + #\@gcccurrent, + #\@dragonegg, + + #\@gcc300, + #\@gcc310, + #\@gcc320, + #\@gcc330, + #\@gcc340, + #\@gcc400, + #\@gcc410, + #\@gcc420, + #\@gcc430, + #\@gcc440, + #\@gcc450, + #\@gcc460, + #\@llvmgcc19, + #\@llvmgcc20, + #\@llvmgcc21, + #\@llvmgcc22, + #\@llvmgcc23, + #\@llvmgcc24, + #\@llvmgcc25, + #\@llvmgcc26, + #\@clang26, + #\@llvmgcc27, + #\@clang27, + #\@llvmgcc28, + #\@clang28, + #\@llvmgcc29, + #\@clang29, + + #\@dcc, + #\@lance, + #\@gcccurrent, + #\@llvmgcc, + #\@clang, + #\@icc, + + #\@gppcurrent, + #\@suncc, + #\@avrgcc, + #\@gcccurrent_O0only, + #\@armcc, + #\@cparser, + #\@llvmgcc24, + #\@llvmgcc_Oonly, + #\@gcccurrent_Oonly, + + #\@gcc_with_ccomp, + #\@clang_with_ccomp, + #\@ccomp, + + ); + +#################################################################### + +# properly parse the return value from system() +sub runit ($) { + my $cmd = shift; + + my $start = new Benchmark; + my $res = (system "$cmd"); + my $end = new Benchmark; + my $dur = timediff($end, $start); + my $exit_value = $? >> 8; + return ($exit_value, $dur); +} + +# build and run the app, with timeouts for both the compiler and the program +sub compile_and_run ($$$$$$$) { + (my $root, my $arch, my $compiler, + my $opt, my $volref, my $wrap_volatiles, my $custom_options) = @_; + my %vols = %{$volref}; + + print "[$arch] $compiler $opt : "; + + my $opt_str = $opt; + ($opt_str =~ s/\ //g); + ($opt_str =~ s/\://g); + ($opt_str =~ s/\-//g); + ($opt_str =~ s/\///g); + if (length($opt_str)>40) { + $opt_str = substr ($opt_str, 0, 40); + } + + my $exe; + my $xtra; + if ($arch eq "ia32" || $arch eq "windiss") { + $exe = "${root}${compiler}${opt_str}_exe"; + ($exe =~ s/\.//g); + $xtra = ""; + } elsif ($arch eq "avr") { + $exe = "${root}${compiler}${opt_str}.elf"; + $xtra = "-mmcu=atmega128 -fwhole-program"; + } elsif ($arch eq "msp" ) { + $exe = "${root}${compiler}${opt_str}.elf"; + $xtra = "-mmcu=msp430x1611"; + } elsif ($arch eq "arm" ) { + $exe = "${root}${compiler}${opt_str}.axf"; + $xtra = "--cpu=ARM926EJ-S -L--datacompressor=off -W"; + } else { + die; + } + + my $srcfile = "$root.c"; + + if ($compiler eq "ccomp") { + $xtra .= " -DINLINE="; + $srcfile = "${root}_small.c"; + } else { + $xtra .= " -w -DINLINE="; + } + + my $wrapstr; + if ($wrap_volatiles) { + $wrapstr = "-DWRAP_VOLATILES=1"; + } else { + $wrapstr = "-DWRAP_VOLATILES=0"; + } + + my $valgrind = ""; + if ($VALGRIND_ON_COMPILER) { + $valgrind = "/home/regehr/valgrind-inst/bin/valgrind --trace-children=yes"; + } + + my $out = "${exe}.out"; + my $compilerout = "${exe}_compiler.out"; + + # ccomp doesn't allow -I option appearing after $srcfile + #my $command = "RunSafely.sh $COMPILER_TIMEOUT 1 /dev/null $compilerout $valgrind $compiler $opt $xtra $srcfile -o $exe $custom_options $wrapstr $notmp -I${CSMITH_PATH}/runtime"; + my $command = "RunSafely.sh $COMPILER_TIMEOUT 1 /dev/null $compilerout $valgrind $compiler $opt $xtra $xxtra $notmp -I${CSMITH_PATH}/runtime $srcfile -o $exe $custom_options $wrapstr $notmp "; + + print "$command\n"; + + (my $res, my $dur) = runit ($command); + + if (($res != 0) || (!(-e $exe))) { + if ($res == $COMPILER_TIMEOUT_RES) { + print STDERR "COMPILER FAILURE: TIMEOUT\n"; + } else { + print STDERR "COMPILER FAILURE with return code $res; output is:\n"; + open INF, "<$compilerout" or die; + while (my $line = ) { print " $line"; } + close INF; + } + return (-2,"",-1); + } + + if (1) { + system "cat $compilerout >> ${VOLATILE_PATH}/compiler_output.txt"; + } + + if ($VALGRIND_ON_COMPILER) { + open INF, "<$compilerout" or die; + while (my $line = ) { + if ($line =~ /ERROR SUMMARY: ([0-9]+) /) { + my $errcount = $1; + print "$line"; + if ($errcount > 0) { + print "SAFETY FAILURE: $line"; + } + } + } + close INF; + } + + if (!$RUN_PROGRAM) { + return (0,"",0); + } + + ($res, $dur) = runit ("run_program.pl $exe $srcfile $arch $CHECK_VOLATILE $compiler > $out"); + + if (-f "vol_addr.txt") { + system("mv vol_addr.txt ${exe}_vol_addr.txt"); + } + + if ($res != 0) { + print "couldn't compute access summary\n"; + return (-1,"",-1); + } + + system "grep 'cpu time' ${exe}.raw-out.time"; + + my $result = ""; + my $code_size = 0; + + open INF, "<$out" or die; + while (my $line = ) { + if ($line =~ /CODE SIZE ([0-9]+)$/) { + $code_size = $1; + } else { + $result .= $line; + } + } + close INF; + + die if (!defined($code_size)); + + return (0,$result,$code_size); + } + +# return a hash whose keys are the names of volatile variables +sub find_volatiles ($) { + (my $cfile) = @_; + + open INF, "<$cfile.c" or die; + my %vols; + my $vcount = 0; + while (my $line = ) { + chomp $line; + if ($line =~ /\/\/ VOLATILE GLOBAL (.+)$/) { + my $v = $1; + $vols{$v} = 1; + $vcount++; + print "volatile $v\n"; + } + } + close INF; + + print "$vcount volatile variables\n"; + + return ($vcount, \%vols); + } + +sub instantiate_test_scripts ($$$$) { + (my $infn, my $opt1, my $opt, my $base_compiler) = @_; + open INF, "<$infn" or die; + open OUTF, ">./test1.sh" or die; + while () { + (s/XX_WORKING_OPT/$opt1/g); + (s/XX_BROKEN_OPT/$opt/g); + if ($base_compiler eq "llvm") { + (s/XX_COMPILER/llvm-gcc/g); + } elsif ($base_compiler eq "gcc") { + (s/XX_COMPILER/gcc/g); + } elsif ($base_compiler eq "armcc") { + # nothing + } else { + die; + } + print OUTF; + } + close INF; + close OUTF; + open INF, "<$infn" or die; + open OUTF, ">./test2.sh" or die; + while () { + (s/XX_WORKING_OPT/$opt1/g); + (s/XX_BROKEN_OPT/$opt/g); + (s/XX_COMPILER/\$1/g); + print OUTF; + } + close INF; + close OUTF; + runit ("chmod a+x ./test1.sh") || die; + runit ("chmod a+x ./test2.sh") || die; +} + +my $lockmgr; + +sub lockit() { + $lockmgr = LockFile::Simple->make ( + -autoclean => 1, + -max => 10, + -nfs => 1, + -hold => 15000, + -stale => 1, + ); + my $res = $lockmgr->lock($LOCKFN); + if (!$res) { + print "couldn't get lockfile to build compiler -- not doing triage on this bug\n"; + } + return $res; +} + +sub unlockit() { + $lockmgr->unlock($LOCKFN); +} + +sub triage ($$$$$$$) { + (my $compiler, my $templ, + my $opt1, my $opt, + my $root, my $base_compiler, + my $version_searchp, my $arch) = @_; + return if (!$DO_TRIAGE); + return if ( + $base_compiler ne "gcc" && + $base_compiler ne "llvm" && + $base_compiler ne "armcc"); + if (lockit()) { + my $res; + print "diffing $opt1 and $opt\n"; + runit "cp $root.c small.c" or die; + instantiate_test_scripts ($templ, $opt1, $opt, $base_compiler); + if ($VERSION_SEARCH) { + if ($base_compiler ne "armcc" && + $arch ne "avr") { + $res = runit ("version_search.pl $base_compiler"); + } + } + runit ("reduce_miscompile.sh > reduce_log.txt 2>&1"); + print ("reduced size: "); + runit ("wc -c small.c"); + if ($VERSION_SEARCH) { + if ($base_compiler ne "armcc" && + $arch ne "avr") { + $res = runit ("version_search.pl $base_compiler"); + } + } + unlockit(); + } + } + +sub test_compiler ($$$$$) { + (my $root, my $compiler_ref, my $volsref, my $wrap, + my $custom_options) = @_; + + (my $arch, my $base_compiler, my $compiler, my $optref) = @{$compiler_ref}; + my @OPTS = @{$optref}; + + my $undef; + + print "--------------------\n"; + + if ($SKIP) { + if ((-f "../../${NOTEFILE_PREFIX}checksum_$compiler.txt") && + (-f "../../${NOTEFILE_PREFIX}checksum_$compiler.txt")) { + print "skipping $compiler\n"; + return (0, 0, 0, "f", 1); + } + } + + my %results; + my %csums; + my $min_code_size = 100000000; + my $success = 0; + my $compiler_fail = 0; + + my %var_reads; + my %var_writes; + my %num_reads; + my %num_writes; + my $first = 1; + + foreach my $opt (@OPTS) { + (my $res, my $res_str, my $code_size) = + compile_and_run ($root, $arch, $compiler, + $opt, $volsref, $wrap, $custom_options); + $num_reads{$opt} = 0; + $num_writes{$opt} = 0; + if ($res == 0) { + $success++; + if ($RUN_PROGRAM) { + if ($code_size < $min_code_size) { + $min_code_size = $code_size; + } + my $checksum_regex = "checksum = (TIMEOUT|[0-9a-fA-F]+)\\s*"; + die if ((!($res_str =~ s/$checksum_regex//)) && + (!($res_str =~ "TIMEOUT"))); + my $csum = $1; + print "$res_str"; + print "checksum = $csum\n"; + $results{$opt} = $res_str; + $csums{$opt} = $csum; + + my $tot_reads = 0; + my $tot_writes = 0; + + while ($res_str =~ /([0-9a-zA-Z\_]+): ([0-9]+) reads, ([0-9]+) writes/g) { + $var_reads{$opt}{$1} = $2; + $var_writes{$opt}{$1} = $3; + $num_reads{$opt} += $2; + $num_writes{$opt} += $3; + $tot_reads += $2; + $tot_writes += $3; + } + + if ($first && + $BAIL_ON_ZERO_WRITES && + $tot_writes == 0) { + print "BAILING due to zero volatile writes\n"; + return (1, 0, $undef, $undef, 1); + } + $first = 0; + + } + } elsif ($res == -2) { + $compiler_fail++; + } else { + die if ($res != -1); + return (1, 0, $undef, $undef); + } + } + + my $result; + my $csum; + my $writes; + my $interesting = 0; + + if ($compiler_fail > 0) { + print "COMPILER FAILED $compiler WRAP=$wrap\n"; + system "touch ../../${NOTEFILE_PREFIX}crash_$compiler.txt"; + $interesting = 1; + } + + if ($success > 0) { + + my $consistent = 1; + my $opt1; + + print "CODE SIZE $compiler WRAP=$wrap ${min_code_size}\n"; + + foreach my $opt (keys %results) { + + if (defined($result)) { + if (($csum ne $csums{$opt}) && + ($csum ne "TIMEOUT" && $csums{$opt} ne "TIMEOUT")) { + print "INTERNAL CHECKSUM FAILURE $compiler $opt WRAP=$wrap\n"; + system "touch ../../${NOTEFILE_PREFIX}checksum_$compiler.txt"; + $interesting = 1; + triage($compiler, + "$VOLATILE_PATH/test-${arch}-${base_compiler}-wrong-code-template.sh", + $opt1, $opt, $root, $base_compiler, $arch); + $consistent = 0; + last; + } + if ($CHECK_VOLATILE) { + if ($result ne $results{$opt}) { + my $write_problem = ""; + if ($num_writes{$opt} != $writes) { + $write_problem = " WRITES"; + } + print "INTERNAL VOLATILE FAILURE${write_problem} $compiler $opt WRAP=$wrap\n"; + $interesting = 1; + triage($compiler, + "$VOLATILE_PATH/test-${arch}-${base_compiler}-volatile-template.sh", + $opt1, $opt, $root, $base_compiler, $arch); + $consistent = 0; + last; + } + } + } else { + $writes = $num_writes{$opt}; + $opt1 = $opt; + $result = $results{$opt}; + $csum = $csums{$opt}; + } + } + + return (0, $consistent, $result, $csum, $interesting); + + } else { + return (0, 0, $result, $csum, $interesting); + } + } + +sub test_program ($$) { + (my $root, my $custom_options) = @_; + + my $vcount; + my $volsref; + my %mt; + + if ($CHECK_VOLATILE == $USE_VOLCHECK) { + ($vcount, $volsref) = find_volatiles ($root); + } else { + $volsref = \%mt; + } + + my @wraps; + if ($TRY_WRAP) { + @wraps = (0,1); + } else { + @wraps = (0); + } + + my $result; + my $csum; + + my $interesting = 0; + + foreach my $compiler_ref (@compilers_to_test) { + foreach my $wrap (@wraps) { + (my $abort_test, my $consistent, my $tmp_result, my $tmp_csum, my $tmp_interesting) = + test_compiler ($root, $compiler_ref, $volsref, $wrap, + $custom_options); + return -1 if ($abort_test != 0); + + if ($tmp_interesting) { + $interesting = 1; + } + + (my $arch, my $base_compiler, my $compiler, my $optref) = @{$compiler_ref}; + print "COMPLETED TEST $compiler WRAP=$wrap\n"; + + # ignore internally inconsistent results + if ($consistent) { + if (defined ($result) && + defined ($csum)) { + if ($CHECK_VOLATILE) { + if ($result ne $tmp_result) { + print "EXTERNAL VOLATILE FAILURE\n"; + $interesting = 1; + } + } + if (($csum ne $tmp_csum) && + ($csum ne "TIMEOUT") && + ($tmp_csum ne "TIMEOUT")) { + print "EXTERNAL CHECKSUM FAILURE\n"; + $interesting = 1; + } + } else { + $result = $tmp_result; + $csum = $tmp_csum; + } + } + } + } + + if ($interesting) { + return 1; + } else { + return 0; + } +} + +#################################################################### + +# on darwin timeout res is 152 instead of 137 +if ($OSNAME =~ /darwin/) { + $COMPILER_TIMEOUT_RES = 152; +} + +die "expecting filename" if scalar(@ARGV < 1); + +my $fn = $ARGV[0]; + +my $custom_options = ""; +if (scalar(@ARGV)==2) { + $custom_options = $ARGV[1]; +} + +my $res = test_program ($fn, $custom_options); +exit ($res); + +#################################################################### + diff --git a/utah/scripts/john_driver/kill-random.pl b/utah/scripts/john_driver/kill-random.pl new file mode 100755 index 000000000..d358e5d64 --- /dev/null +++ b/utah/scripts/john_driver/kill-random.pl @@ -0,0 +1,13 @@ +#!/usr/bin/perl -w + +use strict; + +open INF, "ps axu |" or die; +while () { + next if (!(/random\_test/)); + my @l = split /\s+/; + my $pid = $l[1]; + print "killing $pid\n"; + system "kill -9 $pid"; +} +close INF; diff --git a/utah/scripts/john_driver/launchn.pl b/utah/scripts/john_driver/launchn.pl new file mode 100755 index 000000000..723e78818 --- /dev/null +++ b/utah/scripts/john_driver/launchn.pl @@ -0,0 +1,49 @@ +#!/usr/bin/perl -w + +# add options to backup or delete existing data + +use strict; +use warnings; +use Sys::CPU; + +sub usage () { + die "usage: launchn.pl [number] [seedfile]\n"; +} + +my $SEEDFILE = ""; +my $CPUS; +my $argc = scalar (@ARGV); + +if ($argc == 0) { + $CPUS = Sys::CPU::cpu_count(); + print "looks like we have $CPUS cpus\n"; +} elsif ($argc >= 1) { + $CPUS = $ARGV[0]; + usage() if (!($CPUS =~ /^[0-9]+$/)); + if ($argc == 2) { + $SEEDFILE = $ARGV[1]; + } else { + usage(); + } +} else { + usage(); +} + +open INF, "ps axu |" or die; +while () { + if (/random\_test/) { + print "oops-- kill existing random_test.pl processes first!\n"; + exit; + } +} +close INF; + +system "nohup random_test_llvm_killer.pl > llvm_killer.log 2>&1 &"; + +for (my $i=0; $i<$CPUS; $i++) { + my $dir = "work$i"; + system "rm -rf $dir"; + system "mkdir $dir"; + system "env RVDEBUG_HOME=/home/regehr/z/tmp/arm_tmp$i nice -19 nohup random_test.pl $dir $SEEDFILE > $dir/output.txt 2>&1 &"; +} + diff --git a/utah/scripts/john_driver/random_test.pl b/utah/scripts/john_driver/random_test.pl new file mode 100755 index 000000000..e0572c9ed --- /dev/null +++ b/utah/scripts/john_driver/random_test.pl @@ -0,0 +1,257 @@ +#!/usr/bin/perl -w + +use strict; +use Digest::MD5 qw(md5 md5_hex md5_base64); +use File::stat; + +# when errors are detected, fork off reduction scripts + +################################################################## + +my $SAVE_BADS = 0; + +my $MIN_PROGRAM_SIZE = 20000; +my $MAX_PROGRAM_SIZE = 50000; + +my $CSMITH_TIMEOUT = 180; + +my $PROVIDE_SEED = 1; + +my $XTRA = ""; +$XTRA .= " --force-globals-static "; +#$XTRA .= "--no-argc"; +#$XTRA .= "--concise "; +#$XTRA .= "--no-paranoid "; +#$XTRA .= "--random-random "; +#$XTRA .= "--math-notmp "; + +my $BF = " --bitfields "; +#my $BF = ""; + +#my $PACK = "--packed-struct"; +my $PACK = ""; + +my $QUIET = "--quiet"; +#my $QUIET = ""; + +my $notmp = "-DUSE_MATH_MACROS_NOTMP"; +#my $notmp = ""; + +my $PINTOOL_VOL_ADDR = "vol_addr.txt"; + +# my $platform = "x86"; +my $platform = "x86_64"; + +# remove the comment below to enable ccomp test +#my $CSMITH_CCOMP = "$BF --quiet --no_return_dead_ptr --no-math64 --no-volatiles --ccomp --math-notmp"; +# my $CSMITH_CCOMP = "$BF --quiet --enable-volatile-tests x86 --vol-addr-file $PINTOOL_VOL_ADDR --no-math64 --ccomp --max-array-dim 3 --max-array-len-per-dim 5 --max-struct-fields 5 --math-notmp"; + +my $CSMITH_CCOMP = ""; + +# set up pintool for volatile testing +my $use_pintool = 0; +# my $use_pintool = 1; + +#my $PIN_MODE = "-output_mode verbose"; # We are not supporting it right now +my $PIN_MODE = ""; + +if ($use_pintool) { + # Before you could use pintool to test volatile accesses, change the pintool location when necessary! + my $PIN_PATH = $ENV{"PIN_PATH"}; + die "oops: PIN_PATH environment variable needs to be set" + if (!defined($PIN_PATH)); + + $XTRA .= " --enable-volatile-tests $platform --vol-addr-file $PINTOOL_VOL_ADDR "; + my $pin_cmd; + + # make sure we are going to use the correct version of pinatrace.so, + # i.e., obj-ia32/pinatrace.so for x86 and obj-intel64/pinatrace.so for x86_64. + # For testing compcert, we use the ia32 version + if (($platform eq "x86") || (not ($CSMITH_CCOMP eq ""))) { + $pin_cmd = "$PIN_PATH/ia32/bin/pinbin -t $PIN_PATH/source/tools/ManualExamples/obj-ia32/pinatrace.so -vol_input $PINTOOL_VOL_ADDR $PIN_MODE --"; + } + elsif ($platform eq "x86_64") { + $pin_cmd = "$PIN_PATH/intel64/bin/pinbin -t $PIN_PATH/source/tools/ManualExamples/obj-intel64/pinatrace.so -vol_input $PINTOOL_VOL_ADDR $PIN_MODE --"; + } + else { + die "Invalid platform[$platform] for pintool!"; + } + + $ENV{"PIN_CMD"} = $pin_cmd; +} + +################################################################## + +my $CSMITH_PATH = $ENV{"CSMITH_PATH"}; + +my $good = 0; + +my %checksums; + +my $TIMED_TEST; + +# properly parse the return value from system() +sub runit ($) { + my $cmd = shift; + my $res = (system "$cmd"); + my $exit_value = $? >> 8; + return $exit_value; +} + +sub doit ($$) { + (my $n, my $work) = @_; + print "------ RANDOM PROGRAM $n ------\n"; + if ($TIMED_TEST || ($n%100)==0) { + print "timestamp: "; + system "date"; + } + my $nstr = sprintf "%06d", $n; + my $dir = "${work}/$nstr"; + + system "mkdir $dir"; + chdir $dir; + + my $fn = "rand$nstr"; + my $cfile = "${fn}.c"; + + my $SEED = ""; + if ($PROVIDE_SEED) { + my $n = int (rand (2147483647)); + $SEED = "-s $n"; + print "seed = $n\n"; + } + if ($TIMED_TEST) { + die if ($PROVIDE_SEED); + my $line = ; + if (!$line) { + print "input exhausted, exiting.\n"; + exit (0); + } + chomp $line; + printf "seedfile line: $line\n"; + die if (!($line =~ /\(([0-9]+)\,([0-9]+)\,([0-9]+)\)/)); + $SEED = "-s $1 --max-block-size $2 --max-funcs $3"; + } + + my $cmd; + if ($CSMITH_CCOMP eq "") { + $cmd = "$CSMITH_PATH/src/csmith $SEED $BF $PACK $XTRA --output $cfile"; + } + else { + $cmd = "$CSMITH_PATH/src/csmith $SEED $CSMITH_CCOMP --output $cfile"; + } + if ($PROVIDE_SEED) { + print "$cmd\n"; + } + my $res = runit ("RunSafely.sh $CSMITH_TIMEOUT 1 /dev/zero csmith_output.txt $cmd"); + + if ($res != 0 || !(-e "$cfile")) { + print "CSMITH FAILED\n"; + system "cat csmith_output.txt"; + chdir "../.."; + system "rm -rf $dir"; + return; + } + + my $filesize = stat("$cfile")->size; + print "$cfile is $filesize bytes\n"; + if (($filesize < $MIN_PROGRAM_SIZE) && !$TIMED_TEST) { + print "FILE TOO SMALL\n"; + chdir "../.."; + system "rm -rf $dir"; + return; + } + + my $seed; + my $prog = ""; + my $vcount = 0; + + open INF, "<$cfile" or die; + while (my $line = ) { + if ($line =~ /volatile/) { + $vcount++; + } + if ($line =~ /Seed:\s+([0-9]+)$/) { + $seed = $1; + } + chomp $line; + $prog .= "$line "; + } + close INF; + die if (!defined($seed)); + if (!$PROVIDE_SEED) { + print "regenerate with: '$cmd -s $seed'\n"; + } + + ($prog =~ s/\/\*(.*?)\*\///g); + my $digest = md5($prog); + if ($checksums{$digest}) { + print "BAD PROGRAM: DUPLICATE\n"; + chdir "../.."; + system "rm -rf $dir"; + return; + } + $checksums{$digest} = 1; + + if ($vcount < 1) { + print "NOT ENOUGH VOLATILES\n"; + chdir "../.."; + system "rm -rf $dir"; + return; + } + + if ($CSMITH_CCOMP ne "") { + # ccomp doesn't like asserts, regenerate random programs without asserts. + my $noparanoid_cfile = "${fn}_small.c"; + my $cmd1 = "$CSMITH_PATH/src/csmith $CSMITH_CCOMP -s $seed --output $noparanoid_cfile"; + my $res1 = runit ("RunSafely.sh $CSMITH_TIMEOUT 1 /dev/zero randprog_output.txt $cmd1"); + } + + my $ret = system "evaluate_program.pl $fn"; + my $rc = ($ret>>8) & 0xff; + + chdir "../.."; + + #print "evaluate program returned $rc\n"; + + if ($rc == 0 || $rc == 1) { + print "GOOD PROGRAM: number $good\n"; + $good++; + } else { + print "BAD PROGRAM: doesn't count towards goal.\n"; + } + + # if ($rc != 1 && !$SAVE_BADS) { + if (1) { + system "rm -rf $dir"; + } +} + +if (scalar(@ARGV) != 1 && + scalar(@ARGV) != 2) { + die "usage: random_test.pl work_dir [seedfile]"; +} + +my $work = $ARGV[0]; + +if (!(-d $work)) { + die "error: create work dir '$work' first"; +} + +if (scalar(@ARGV)==2) { + my $seedfile = $ARGV[1]; + open SEEDFILE, "<$seedfile" or die "error: cannot open seed file $seedfile"; + $TIMED_TEST = 1; + $PROVIDE_SEED = 0; +} else { + $TIMED_TEST = 0; +} + +my $i = 0; +while (1) { + doit ($i, $work); + $i++; + print "\n"; +} + +################################################################## diff --git a/utah/scripts/john_driver/see_results.sh b/utah/scripts/john_driver/see_results.sh new file mode 100755 index 000000000..a6ad82fa3 --- /dev/null +++ b/utah/scripts/john_driver/see_results.sh @@ -0,0 +1,2 @@ +grep FAIL work*/output.txt | grep -v 'COMPILER FAILURE' +grep GOOD work*/output.txt | wc