Permalink
Browse files

Make Gradle dependencies available offline (#131)

* Added a script to extract the URL of all gradle distributions used in the vcs history.

* Download all Gradle dependencies in the init script.

* Modify gradle-wrapper.properties when checking out a Mockito version: set URL to a local file.

* Allow test_verify_bugs script to run on a subset of all bugs for a given project.

* Added build system dependencies to .gitignore.

* Download build system dependencies in init script.

* Make use of the range operator in travis script.

* Update gradle.properties file if it exists.

* Issue a warning if the build systems are not initialized.
  • Loading branch information...
rjust committed Jan 18, 2018
1 parent 1883d7c commit 107ea67191fa9bb6688fc2e3ffb4da422852af13
View
@@ -2,4 +2,5 @@ project_repos/
major/
framework/test/*.log
framework/lib/test_generation
framework/lib/build_systems
work/
View
@@ -5,16 +5,46 @@ perl:
jdk:
- openjdk7
addons:
apt:
packages:
- openjdk-7-jdk
install:
- jdk_switcher use openjdk7
- (echo y;echo o conf prerequisites_policy follow;echo o conf commit)|cpan
- export PATH=$JAVA_HOME/bin:$PATH
- (echo y;echo o conf prerequisites_policy follow;echo o conf commit) | cpan
- cpan install Carton
- carton install
- ./init.sh
script:
- cd framework/test
- carton exec ./test_tutorial.sh
- carton exec ./test_mutation_analysis.sh
- carton exec ./test_randoop.sh
- carton exec ./test_fix_test_suite.sh
jobs:
include:
# Test the tutorial and basic commands
- script: carton exec ./test_tutorial.sh
- script: carton exec ./test_mutation_analysis.sh
- script: carton exec ./test_randoop.sh
- script: carton exec ./test_fix_test_suite.sh
# Verify that all bugs are reproducible (run multiple jobs for projects that
# take a long time to finish.
- stage: verify
## Chart
script: carton exec ./test_verify_bugs.sh -p Chart
## Lang
- script: carton exec ./test_verify_bugs.sh -p Lang -b1..32
- script: carton exec ./test_verify_bugs.sh -p Lang -b33..65
## Math
- script: carton exec ./test_verify_bugs.sh -p Math -b1..27
- script: carton exec ./test_verify_bugs.sh -p Math -b28..54
- script: carton exec ./test_verify_bugs.sh -p Math -b55..80
- script: carton exec ./test_verify_bugs.sh -p Math -b81..106
## Mockito
- script: carton exec ./test_verify_bugs.sh -p Mockito -b1..5
- script: carton exec ./test_verify_bugs.sh -p Mockito -b6..10
- script: carton exec ./test_verify_bugs.sh -p Mockito -b11..15
- script: carton exec ./test_verify_bugs.sh -p Mockito -b16..20
- script: carton exec ./test_verify_bugs.sh -p Mockito -b21..27
- script: carton exec ./test_verify_bugs.sh -p Mockito -b28..38
## Time
- script: carton exec ./test_verify_bugs.sh -p Time
@@ -152,6 +152,16 @@ The directory of the libraries of the test generation tools (I<C<LIB_DIR>/test_g
=cut
our $TESTGEN_LIB_DIR = ($ENV{'TESTGEN_LIB_DIR'} // "$LIB_DIR/test_generation/generation");
=pod
=item C<BUILD_SYSTEMS_LIB_DIR>
The directory of the libraries of the build system tools (I<C<LIB_DIR>/build_systems>)
=cut
our $BUILD_SYSTEMS_LIB_DIR = ($ENV{'BUILD_SYSTEMS_LIB_DIR'} // "$LIB_DIR/build_systems");
=pod
=item C<D4J_BUILD_FILE>
@@ -170,11 +180,13 @@ our $D4J_BUILD_FILE = ($ENV{'D4J_BUILD_FILE'} or "$SCRIPT_DIR/projects/defects4j
# - External libraries (test generation) available?
#
-e "$REPO_DIR/README"
or die("Couldn't find project repositories! Did you run 'defects4j/init.sh'?\n\n");
or die("Couldn't find project repositories! Did you (re)run 'defects4j/init.sh'?\n\n");
-e "$MAJOR_ROOT/bin/ant"
or die("Couldn't find Major mutation framework! Did you run 'defects4j/init.sh'?\n\n");
or die("Couldn't find Major mutation framework! Did you (re)run 'defects4j/init.sh'?\n\n");
-d "$TESTGEN_LIB_DIR"
or die("Couldn't find external libraries! Did you run 'defects4j/init.sh'?\n\n");
or die("Couldn't find test generation tools! Did you (re)run 'defects4j/init.sh'?\n\n");
-d "$BUILD_SYSTEMS_LIB_DIR"
or die("Couldn't find build system tools! Did you (re)run 'defects4j/init.sh'?\n\n");
# Add script and core directory to @INC
unshift(@INC, $CORE_DIR);
@@ -221,6 +233,7 @@ $UTIL_DIR
$BASE_DIR
$MAJOR_ROOT
$TESTGEN_LIB_DIR
$BUILD_SYSTEMS_LIB_DIR
$REPO_DIR
$D4J_TMP_DIR
@@ -70,6 +70,33 @@ sub _post_checkout {
if ($id == 16 || $id == 17 || ($id >= 34 && $id <= 38)) {
$vcs->apply_patch($work_dir, "$mockito_junit_runner_patch_file") or confess("Couldn't apply patch ($mockito_junit_runner_patch_file): $!");
}
# Change Url to Gradle distribution
my $prop = "$work_dir/gradle/wrapper/gradle-wrapper.properties";
my $lib_dir = "$LIB_DIR/build_systems/gradle";
# Read existing Gradle properties file, if it exists
open(PROP, "<$prop") or return;
my @tmp;
while (<PROP>) {
if (/(distributionUrl=).*\/(gradle-2.*)/) {
s/(distributionUrl=).*\/(gradle-.*)/$1file\\:$lib_dir\/gradle-2.2.1-all.zip/g;
} else {
s/(distributionUrl=).*\/(gradle-.*)/$1file\\:$lib_dir\/gradle-1.12-bin.zip/g;
}
push(@tmp, $_);
}
close(PROP);
# Update properties file
open(OUT, ">$prop") or die "Cannot write properties file";
print(OUT @tmp);
close(OUT);
# Disable the Gradle daemon
if (-e "$work_dir/gradle.properties") {
system("sed -i.bak s/org.gradle.daemon=true/org.gradle.daemon=false/g \"$work_dir/gradle.properties\"");
}
}
#
@@ -79,10 +79,14 @@
<delete file="${basedir}/buildSrc/src/test/groovy/org/mockito/release/notes/improvements/GitHubTicketFetcherTest.groovy" quiet="true" />
<!-- Call the Gradle build script -->
<exec executable="/bin/bash" failonerror="true">
<arg value="gradlew"/>
<exec executable="${basedir}/gradlew" failonerror="true">
<arg value="-xtestClasses"/>
<arg value="-xtest"/>
<arg value="classes"/>
</exec>
<exec executable="${basedir}/gradlew" failonerror="true">
<arg value="--stop"/>
</exec>
</target>
<!-- Compile the tests using Gradle, used in newer commits -->
@@ -91,10 +95,13 @@
<delete file="${basedir}/buildSrc/src/test/groovy/org/mockito/release/notes/improvements/GitHubTicketFetcherTest.groovy" quiet="true" />
<!-- Call the Gradle build script -->
<exec executable="/bin/bash" failonerror="true">
<arg value="gradlew"/>
<exec executable="${basedir}/gradlew" failonerror="true">
<arg value="-xtest"/>
<arg value="testClasses"/>
</exec>
<exec executable="${basedir}/gradlew" failonerror="true">
<arg value="--stop"/>
</exec>
</target>
<!-- Mutate classes if mutation is enabled -->
@@ -109,12 +116,13 @@
<arg value="${d4j.home}"/>
</exec>
<!-- Call the Gradle build script -->
<exec executable="/bin/bash" failonerror="true">
<arg value="gradlew"/>
<arg value="classes"/>
<exec executable="${basedir}/gradlew" failonerror="true">
<arg value="-xtestClasses"/>
<arg value="-xtest"/>
<arg value="-xjar"/>
<arg value="classes"/>
</exec>
<exec executable="${basedir}/gradlew" failonerror="true">
<arg value="--stop"/>
</exec>
</try>
<finally>
@@ -15,7 +15,7 @@ init
# Any version should work, but the test cases below are specific to this version
pid="Lang"
bid="1f"
bid="6f"
pid_bid_dir="$TMP_DIR/$pid-$bid"
rm -rf "$pid_bid_dir"
@@ -70,7 +70,7 @@ defects4j checkout -p "$pid" -v "$bid" -w "$pid_bid_dir" || die "It was not poss
rm -f "$summary_file"
defects4j mutation -w "$pid_bid_dir" -r || die "Mutation analysis (including all mutants) failed!"
_check_mutation_result 941 913 646
_check_mutation_result 42 42 36
###################################################
# Test mutation analysis when excluding all mutants
@@ -83,7 +83,7 @@ exclude_file="$pid_bid_dir/exclude_all_mutants.txt"
cut -f1 -d':' "$mutants_file" > "$exclude_file"
defects4j mutation -w "$pid_bid_dir" -r -e "$exclude_file" || die "Mutation analysis (excluding all mutants) failed!"
_check_mutation_result 941 0 0
_check_mutation_result 42 0 0
##########################################################################
# Test mutation analysis when explicitly providing the class(es) to mutate
@@ -96,7 +96,7 @@ instrument_classes="$pid_bid_dir/instrument_classes.txt"
echo "org.apache.commons.lang3.LocaleUtils" > "$instrument_classes"
defects4j mutation -w "$pid_bid_dir" -i "$instrument_classes" || die "Mutation analysis (instrument LocaleUtils) failed!"
_check_mutation_result 184 184 158
_check_mutation_result 189 189 149
# Clean up
rm -rf "$pid_bid_dir"
@@ -1,23 +1,64 @@
#!/usr/bin/env bash
################################################################################
#
# This script verifies that all bugs in Defects4J are reproducible and that the
# provided information about triggering tests is correct.
# This script verifies that all bugs for a given project are reproducible and
# that the provided information about triggering tests is correct.
#
# Examples for Lang:
# * Verify all bugs: ./test_verify_bugs.sh -pLang
# * Verify bugs 1-10: ./test_verify_bugs.sh -pLang -b1..10
# * Verify bugs 1 and 3: ./test_verify_bugs.sh -pLang -b1 -b3
# * Verify bugs 1-10 and 20: ./test_verify_bugs.sh -pLang -b1..10 -b20
#
################################################################################
# Import helper subroutines and variables, and init Defects4J
source test.include
# Check whether only a subset of projects should be tested
if [ $# -eq 0 ]; then
projects=( Chart Closure Lang Math Mockito Time )
else
projects=( $* )
script_name=$(echo $script | sed 's/\.sh$//')
LOG="$TEST_DIR/${script_name}$(printf '_%s' $*).log"
# Print usage message and exit
usage() {
echo "usage: $0 [-b <bug id> ... | -b <bug id range> ... ] -p <project id>"
exit 1
}
# Check arguments
while getopts ":p:b:" opt; do
case $opt in
p) PID="$OPTARG"
;;
b) if [[ "$OPTARG" =~ ^[1-9]*\.\.[0-9]*$ ]]; then
BUGS="$BUGS $(eval echo {$OPTARG})"
else
BUGS="$BUGS $OPTARG"
fi
;;
\?)
echo "Unknown option: -$OPTARG" >&2
usage
;;
:)
echo "No argument provided: -$OPTARG." >&2
usage
;;
esac
done
if [ "$PID" == "" ]; then
usage
fi
#TODO: Check whether PID is one of {Chart, Closure, Lang, Math, Mockito, Time}
init
# Run all bugs, unless otherwise specified
if [ "$BUGS" == "" ]; then
num_bugs=$(num_lines $BASE_DIR/framework/projects/$PID/commit-db)
BUGS="$(seq 1 1 $num_bugs)"
fi
# Create log file
script_name=$(echo $script | sed 's/\.sh$//')
LOG="$TEST_DIR/${script_name}$(printf '_%s_%s' $PID $$).log"
################################################################################
# Run developer-written tests on all buggy and fixed program versions, and
# verify trigger tests
@@ -29,36 +70,33 @@ HALT_ON_ERROR=0
test_dir="$TMP_DIR/test_trigger"
mkdir -p $test_dir
for pid in "${projects[@]}"; do
num_bugs=$(num_lines $BASE_DIR/framework/projects/$pid/commit-db)
work_dir="$test_dir/$pid"
# Clean working directory
rm -rf $work_dir
for bid in $(seq 1 1 $num_bugs); do
for v in "b" "f"; do
vid=${bid}$v
defects4j checkout -p $pid -v "$vid" -w "$work_dir" || die "checkout: $pid-$vid"
defects4j compile -w "$work_dir" || die "compile: $pid-$vid"
defects4j test -r -w "$work_dir" || die "run relevant tests: $pid-$vid"
triggers=$(num_triggers "$work_dir/failing_tests")
# Expected number of failing tests for each fixed version is 0!
if [ $v == "f" ]; then
[ $triggers -eq 0 ] \
|| die "verify number of triggering tests: $pid-$vid (expected: 0, actual: $triggers)"
continue
fi
# Expected number of failing tests for each buggy version is equal
# to the number of provided triggering tests
expected=$(num_triggers "$BASE_DIR/framework/projects/$pid/trigger_tests/$bid")
[ $triggers -eq $expected ] \
|| die "verify number of triggering tests: $pid-$vid (expected: $expected, actual: $triggers)"
for t in $(get_triggers "$BASE_DIR/framework/projects/$pid/trigger_tests/$bid"); do
grep -q "$t" "$work_dir/failing_tests" || die "verify name of triggering tests ($t not found)"
done
work_dir="$test_dir/$PID"
# Clean working directory
rm -rf $work_dir
for bid in $(echo $BUGS); do
for v in "b" "f"; do
vid=${bid}$v
defects4j checkout -p $PID -v "$vid" -w "$work_dir" || die "checkout: $PID-$vid"
defects4j compile -w "$work_dir" || die "compile: $PID-$vid"
defects4j test -r -w "$work_dir" || die "run relevant tests: $PID-$vid"
triggers=$(num_triggers "$work_dir/failing_tests")
# Expected number of failing tests for each fixed version is 0!
if [ $v == "f" ]; then
[ $triggers -eq 0 ] \
|| die "verify number of triggering tests: $PID-$vid (expected: 0, actual: $triggers)"
continue
fi
# Expected number of failing tests for each buggy version is equal
# to the number of provided triggering tests
expected=$(num_triggers "$BASE_DIR/framework/projects/$PID/trigger_tests/$bid")
[ $triggers -eq $expected ] \
|| die "verify number of triggering tests: $PID-$vid (expected: $expected, actual: $triggers)"
for t in $(get_triggers "$BASE_DIR/framework/projects/$PID/trigger_tests/$bid"); do
grep -q "$t" "$work_dir/failing_tests" || die "verify name of triggering tests ($t not found)"
done
done
rm -rf $work_dir
done
rm -rf $work_dir
HALT_ON_ERROR=1
Oops, something went wrong.

0 comments on commit 107ea67

Please sign in to comment.