Permalink
Browse files

Merge pull request #108 from rjust/fix_mutant_caching

Clean up mutation-related modules and fix issue #87
  • Loading branch information...
rjust committed Aug 14, 2017
2 parents 5ac721a + aa6955b commit fee5ddf020d0ce9c793655b74f0ab068153c03ef
@@ -133,23 +133,17 @@ if ($type eq "b") {
exit 1;
}
# The mutation operators that should be enabled
my @MUT_OPS = ("AOR", "LOR","SOR", "COR", "ROR", "ORU", "LVR", "STD");
# Instantiate project and set working directory
my $project = Project::create_project($pid);
$project->{prog_root} = $WORK_DIR;
# Classes to mutate -- default is all modified classes
my $classes = $INSTRUMENT // "$SCRIPT_DIR/projects/$pid/modified_classes/$bid.src";
# Create mutation definitions (mml file)
my $mml_dir = "$WORK_DIR/mml";
system("$UTIL_DIR/create_mml.pl -p $pid -c $classes -o $mml_dir -b $bid");
my $mml_file = "$mml_dir/${bid}.mml.bin";
-e $mml_file or die "Mml file does not exist: $mml_file!";
# Mutate and compile sources
$ENV{MML} = $mml_file;
$project->mutate() > 0 or die "Cannot mutate project!";
$project->mutate($classes, \@MUT_OPS) > 0 or die "Cannot mutate project!";
my $log_file = "$WORK_DIR/.mutation.log";
# Run the test suite, according to the provided flags
@@ -131,6 +131,9 @@ =head1 DESCRIPTION
# Enable debugging if flag is set
$DEBUG = 1 if defined $cmd_opts{D};
# The mutation operators that should be enabled
my @MUT_OPS = ("AOR", "LOR","SOR", "COR", "ROR", "ORU", "LVR", "STD");
# Set up project
my $project = Project::create_project($PID);
@@ -285,15 +288,7 @@ sub _run_mutation {
close(EXCL_FILE);
}
# Create mutation definitions (mml file)
my $mml_dir = "$TMP_DIR/.mml";
system("$UTIL_DIR/create_mml.pl -p $PID -c $TARGET_CLASSES_DIR/$bid.src -o $mml_dir -b $bid");
my $mml_file = "$mml_dir/$bid.mml.bin";
-e $mml_file or die "Mml file does not exist: $mml_file!";
# Mutate source code
$ENV{MML} = $mml_file;
my $gen_mutants = $project->mutate();
my $gen_mutants = $project->mutate("$TARGET_CLASSES_DIR/$bid.src", \@MUT_OPS);
$gen_mutants > 0 or die "No mutants generated for $vid!";
# Compile generated tests
@@ -196,8 +196,10 @@ our $CONFIG_VID = "vid";
# Filename which stores build properties
our $PROP_FILE = "defects4j.build.properties";
our $PROP_EXCLUDE = "d4j.tests.exclude";
our $PROP_INSTRUMENT = "d4j.classes.instrument";
# Keys of stored properties
our $PROP_EXCLUDE = "d4j.tests.exclude";
our $PROP_INSTRUMENT = "d4j.classes.instrument";
our $PROP_MUTATE = "d4j.classes.mutate";
our $PROP_DIR_SRC_CLASSES = "d4j.dir.src.classes";
our $PROP_DIR_SRC_TESTS = "d4j.dir.src.tests";
our $PROP_CLASSES_MODIFIED= "d4j.classes.modified";
@@ -234,6 +236,7 @@ $CONFIG_VID
$PROP_FILE
$PROP_EXCLUDE
$PROP_INSTRUMENT
$PROP_MUTATE
$PROP_DIR_SRC_CLASSES
$PROP_DIR_SRC_TESTS
$PROP_CLASSES_MODIFIED
View
@@ -54,6 +54,48 @@ my $SUMMARY_FILE = "summary.csv";
=head2 Static subroutines
Mutation::create_mml(instrument_classes, out_file, mut_ops)
Generates an mml file, enabling all mutation operators defined by the array
reference C<mut_ops> for all classes listed in F<instrument_classes>. The mml
(source) file is written to C<out_file>. This subroutine also compiles the mml
file to F<'out_file'.bin>.
=cut
sub create_mml {
@_ == 3 or die $ARG_ERROR;
my ($instrument_classes, $out_file, $mut_ops) = @_;
my $OUT_DIR = Utils::get_dir($out_file);
my $TEMPLATE = `cat $MAJOR_ROOT/mml/template.mml` or die "Cannot read mml template: $!";
system("mkdir -p $OUT_DIR");
open(IN, $instrument_classes);
my @classes = <IN>;
close(IN);
# Generate mml file by enabling operators for listed classes only
open(FILE, ">$out_file") or die "Cannot write mml file ($out_file): $!";
# Add operator definitions from template
print FILE $TEMPLATE;
# Enable operators for all classes
foreach my $class (@classes) {
chomp $class;
print FILE "\n// Enable operators for $class\n";
foreach my $op (@{$mut_ops}) {
# Skip disabled operators
next if $TEMPLATE =~ /-$op<"$class">/;
print FILE "$op<\"$class\">;\n";
}
}
close(FILE);
Utils::exec_cmd("$MAJOR_ROOT/bin/mmlc $out_file 2>&1", "Compiling mutant definition (mml)")
or die "Cannot compile mml file: $out_file!";
}
=pod
Mutation::mutation_analysis(project_ref, log_file [, exclude_file, base_map, single_test])
Runs mutation analysis for the developer-written test suites of the provided
View
@@ -83,6 +83,10 @@ Commons lang (L<Vcs::Git> backend)
Commons math (L<Vcs::Git> backend)
=item * L<Mockito|Project::Mockito>
Mockito (L<Vcs::Git> backend)
=item * L<Time|Project::Time>
Joda-time (L<Vcs::Git> backend)
@@ -96,6 +100,7 @@ use warnings;
use strict;
use Constants;
use Utils;
use Mutation;
use Carp qw(confess);
=pod
@@ -687,14 +692,43 @@ sub coverage_report {
=pod
$project->mutate()
$project->mutate(instrument_classes, mut_ops)
Mutates the checked-out program version.
Mutates all classes listed in F<instrument_classes>, using all mutation operators
defined by the array reference C<mut_ops>, in the checked-out program version.
Returns the number of generated mutants on success, -1 otherwise.
=cut
sub mutate {
my $self = shift;
@_ == 3 or die $ARG_ERROR;
my ($self, $instrument_classes, $mut_ops) = @_;
my $work_dir = $self->{prog_root};
# Read all classes that should be mutated
-e $instrument_classes or die "Classes file ($instrument_classes) does not exist!";
open(IN, "<$instrument_classes") or die "Cannot read $instrument_classes";
my @classes = ();
while(<IN>) {
s/\r?\n//;
push(@classes, $_);
}
close(IN);
# Update properties
my $list = join(",", @classes);
my $config = {$PROP_MUTATE => $list};
Utils::write_config_file("$work_dir/$PROP_FILE", $config);
# Create mutation definitions (mml file)
my $mml_src = "$self->{prog_root}/.mml/default.mml";
my $mml_bin = "${mml_src}.bin";
Mutation::create_mml($instrument_classes, $mml_src, $mut_ops);
-e "$mml_bin" or die "Mml file does not exist: $mml_bin!";
# Set environment variable MML, which is read by Major
$ENV{MML} = $mml_bin;
# Mutate and compile sources
if (! $self->_ant_call("mutate")) {
return -1;
}
View
@@ -37,6 +37,7 @@ use warnings;
use strict;
use File::Basename;
use File::Spec;
use Cwd qw(abs_path);
use Carp qw(confess);
@@ -77,6 +78,20 @@ sub get_abs_path {
return abs_path($dir);
}
=pod
Utils::get_dir(file)
Returns the directory of the absolute path of F<file>.
=cut
sub get_dir {
@_ == 1 or die $ARG_ERROR;
my $path = shift;
my ($volume,$dir,$file) = File::Spec->splitpath($path);
return get_abs_path($dir);
}
=pod
Utils::get_failing_tests(test_result_file)
@@ -13,7 +13,8 @@ HERE=$(cd `dirname $0` && pwd)
source "$HERE/test.include" || exit 1
init
pid="Lang" # any pid-bid should work
# Any version should work, but the test cases below are specific to this version
pid="Lang"
bid="1f"
pid_bid_dir="$TMP_DIR/$pid-$bid"
rm -rf "$pid_bid_dir"
@@ -22,61 +23,80 @@ rm -rf "$pid_bid_dir"
summary_file="$pid_bid_dir/summary.csv"
mutants_file="$pid_bid_dir/mutants.log"
################################################################################
#
# Check whether the mutation analysis results (summary.csv) match the expectations.
#
_check_mutation_result() {
[ $# -eq 3 ] || die "usage: ${FUNCNAME[0]} \
<expected_mutants_generated> \
<expected_mutants_covered> \
<expected_mutants_killed>"
local exp_mut_gen=$1
local exp_mut_cov=$2
local exp_mut_kill=$3
# Make sure Major generated the expected data files
[ -s "$mutants_file" ] || die "'$mutants_file' doesn't exist or is empty!"
[ -s "$summary_file" ] || die "'$summary_file' doesn't exist or is empty!"
# The last row of 'summary.csv' does not have an end of line character.
# Otherwise, using wc would be more intuitive.
local num_rows=$(grep -c "^" "$summary_file")
[ "$num_rows" -eq "2" ] || die "Unexpected number of lines in '$summary_file'!"
# Columns of summary (csv) file:
# MutantsGenerated,MutantsCovered,MutantsKilled,MutantsLive,RuntimePreprocSeconds,RuntimeAnalysisSeconds
local act_mut_gen=$(tail -n1 "$summary_file" | cut -f1 -d',')
local act_mut_cov=$(tail -n1 "$summary_file" | cut -f2 -d',')
local act_mut_kill=$(tail -n1 "$summary_file" | cut -f3 -d',')
[ "$act_mut_gen" -eq "$exp_mut_gen" ] || die "Unexpected number of mutants generated (expected: $exp_mut_gen, actual: $act_mut_gen)!"
[ "$act_mut_cov" -eq "$exp_mut_cov" ] || die "Unexpected number of mutants covered (expected: $exp_mut_cov, actual: $act_mut_cov)!"
[ "$act_mut_kill" -eq "$exp_mut_kill" ] || die "Unexpected number of mutants killed (expected: $exp_mut_kill, actual: $act_mut_kill)!"
# TODO Would be nice to test the number of excluded mutants. In order to do it
# Major has to write that number to the '$pid_bid_dir/summary.csv' file.
}
################################################################################
# Checkout project-version
defects4j checkout -p "$pid" -v "$bid" -w "$pid_bid_dir" || die "It was not possible to checkout $pid-$bid to '$pid_bid_dir'!"
######################################################
# Test mutation analysis without excluding any mutants
defects4j mutation -w "$pid_bid_dir" -r || die "Mutation analysis (including all mutants) failed!"
# Make sure Major generated the expected data files
[ -s "$mutants_file" ] || die "'$mutants_file' doesn't exist or is empty!"
[ -s "$summary_file" ] || die "'$summary_file' doesn't exist or is empty!"
# Remove the summary file to ensure it is regenerated
rm -f "$summary_file"
# The last row of 'summary.csv' does not have an end of line character.
# Otherwise, using wc would be more intuitive.
num_rows=$(grep -c "^" "$summary_file")
[ "$num_rows" -eq "2" ] || die "Unexpected number of lines in '$summary_file'!"
# Columns of summary (csv) file:
# MutantsGenerated,MutantsCovered,MutantsKilled,MutantsLive,RuntimePreprocSeconds,RuntimeAnalysisSeconds
num_mutants_covered=$(tail -n1 "$summary_file" | cut -f2 -d',')
num_mutants_killed=$(tail -n1 "$summary_file" | cut -f3 -d',')
[ "$num_mutants_covered" -gt 0 ] || die "0 mutants covered!"
[ "$num_mutants_killed" -gt 0 ] || die "0 mutants killed!"
# TODO Would be nice to test the number of excluded mutants. In order to do it
# Major has to write that number to the '$pid_bid_dir/summary.csv' file.
# Remove the summary file to ensure it is regenerated later
rm "$summary_file"
defects4j mutation -w "$pid_bid_dir" -r || die "Mutation analysis (including all mutants) failed!"
_check_mutation_result 941 913 646
###################################################
# Test mutation analysis when excluding all mutants
# Remove the summary file to ensure it is regenerated
rm -f "$summary_file"
# Exclude all generated mutants
exclude_file="$pid_bid_dir/exclude_all_mutants.txt"
cut -f1 -d':' "$mutants_file" > "$exclude_file"
defects4j mutation -w "$pid_bid_dir" -r -e "$exclude_file" || die "Mutation analysis (excluding all mutants) failed!"
_check_mutation_result 941 0 0
# The last row of 'summary.csv' does not have an end of line character.
# Otherwise, using wc would be more intuitive.
num_rows=$(grep -c "^" "$summary_file")
[ "$num_rows" -eq "2" ] || die "Unexpected number of lines in '$summary_file'!"
##########################################################################
# Test mutation analysis when explicitly providing the class(es) to mutate
# Columns of summary (csv) file:
# MutantsGenerated,MutantsCovered,MutantsKilled,MutantsLive,RuntimePreprocSeconds,RuntimeAnalysisSeconds
num_mutants_covered=$(tail -n1 "$summary_file" | cut -f2 -d',')
num_mutants_killed=$(tail -n1 "$summary_file" | cut -f3 -d',')
# Remove the summary file to ensure it is regenerated
rm -f "$summary_file"
[ "$num_mutants_covered" -eq 0 ] || die "$num_mutants_covered mutants have been covered!"
[ "$num_mutants_killed" -eq 0 ] || die "$num_mutants_killed mutants have been killed!"
# Mutate an arbitrary, non-modified class
instrument_classes="$pid_bid_dir/instrument_classes.txt"
echo "org.apache.commons.lang3.LocaleUtils" > "$instrument_classes"
# TODO Would be nice to test the number of excluded mutants. In order to do it
# Major has to write that number to the '$pid_bid_dir/summary.csv' file.
defects4j mutation -w "$pid_bid_dir" -i "$instrument_classes" || die "Mutation analysis (instrument LocaleUtils) failed!"
_check_mutation_result 184 184 158
# Clean up
rm -rf "$pid_bid_dir"
Oops, something went wrong.

0 comments on commit fee5ddf

Please sign in to comment.