From 97379d8f64c7f6f0209c2db97dae84bc8d32028d Mon Sep 17 00:00:00 2001 From: Prashant Sharma Date: Fri, 4 Apr 2014 06:25:00 +0530 Subject: [PATCH 1/4] HOTFIX for broken CI, by SPARK-1336 --- .rat-excludes | 1 + dev/run-tests | 9 ++++----- dev/scalastyle | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.rat-excludes b/.rat-excludes index 17cf6d0ed1cf3..85bfad60fcadc 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -39,3 +39,4 @@ work .*\.q golden test.out/* +.*iml diff --git a/dev/run-tests b/dev/run-tests index fff949e04fcd7..91b1bcce6e3dc 100755 --- a/dev/run-tests +++ b/dev/run-tests @@ -26,13 +26,13 @@ rm -rf ./work # Fail fast set -e - +set -o pipefail if test -x "$JAVA_HOME/bin/java"; then declare java_cmd="$JAVA_HOME/bin/java" else declare java_cmd=java fi - +FAILED=0 JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q') [ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run because JDK version is < 1.8." @@ -49,7 +49,7 @@ dev/scalastyle echo "=========================================================================" echo "Running Spark unit tests" echo "=========================================================================" -sbt/sbt assembly test +echo -e "q\n" | sbt/sbt assembly test | grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including" echo "=========================================================================" echo "Running PySpark tests" @@ -63,5 +63,4 @@ echo "=========================================================================" echo "Detecting binary incompatibilites with MiMa" echo "=========================================================================" ./bin/spark-class org.apache.spark.tools.GenerateMIMAIgnore -sbt/sbt mima-report-binary-issues | grep -v -e "info.*Resolving" - +echo -e "q\n" | sbt/sbt mima-report-binary-issues | grep -v -e "info.*Resolving" diff --git a/dev/scalastyle b/dev/scalastyle index 5a18f4d672825..54961a476348a 100755 --- a/dev/scalastyle +++ b/dev/scalastyle @@ -17,8 +17,8 @@ # limitations under the License. # -sbt/sbt clean scalastyle > scalastyle.txt -ERRORS=$(cat scalastyle.txt | grep -e "error file") +echo -e "q\n" | sbt/sbt clean scalastyle > scalastyle.txt +ERRORS=$(cat scalastyle.txt | grep -e "error") if test ! -z "$ERRORS"; then echo -e "Scalastyle checks failed at following occurrences:\n$ERRORS" exit 1 From 7fffdf2d2ed20dc8d13bd7880df9f523ef6824f6 Mon Sep 17 00:00:00 2001 From: Prashant Sharma Date: Fri, 4 Apr 2014 06:29:20 +0530 Subject: [PATCH 2/4] Removed a stray line. --- dev/run-tests | 1 - 1 file changed, 1 deletion(-) diff --git a/dev/run-tests b/dev/run-tests index 91b1bcce6e3dc..208a5e77f75b6 100755 --- a/dev/run-tests +++ b/dev/run-tests @@ -32,7 +32,6 @@ if test -x "$JAVA_HOME/bin/java"; then else declare java_cmd=java fi -FAILED=0 JAVA_VERSION=$($java_cmd -version 2>&1 | sed 's/java version "\(.*\)\.\(.*\)\..*"/\1\2/; 1q') [ "$JAVA_VERSION" -ge 18 ] && echo "" || echo "[Warn] Java 8 tests will not run because JDK version is < 1.8." From f8659512e13c610e4a77277ebe9274cc3d27f6df Mon Sep 17 00:00:00 2001 From: Prashant Sharma Date: Fri, 4 Apr 2014 06:58:17 +0530 Subject: [PATCH 3/4] made error to match with word boundry so errors does not match. This is there to make sure build fails if provided SparkBuild has compile errors. --- dev/scalastyle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/scalastyle b/dev/scalastyle index 54961a476348a..19955b9aaaad3 100755 --- a/dev/scalastyle +++ b/dev/scalastyle @@ -18,7 +18,7 @@ # echo -e "q\n" | sbt/sbt clean scalastyle > scalastyle.txt -ERRORS=$(cat scalastyle.txt | grep -e "error") +ERRORS=$(cat scalastyle.txt | grep -e "\") if test ! -z "$ERRORS"; then echo -e "Scalastyle checks failed at following occurrences:\n$ERRORS" exit 1 From 9d22bc2a2aaf33cf044a76f1b893ee95c97884ab Mon Sep 17 00:00:00 2001 From: Prashant Sharma Date: Sat, 5 Apr 2014 09:52:35 +0530 Subject: [PATCH 4/4] added comment why echo -e q exists. --- dev/run-tests | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dev/run-tests b/dev/run-tests index 208a5e77f75b6..6ad674a2ba127 100755 --- a/dev/run-tests +++ b/dev/run-tests @@ -48,6 +48,8 @@ dev/scalastyle echo "=========================================================================" echo "Running Spark unit tests" echo "=========================================================================" +# echo "q" is needed because sbt on encountering a build file with failure (either resolution or compilation) +# prompts the user for input either q, r, etc to quit or retry. This echo is there to make it not block. echo -e "q\n" | sbt/sbt assembly test | grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including" echo "========================================================================="