From 791697f9bb62e84741263edde7c4ba88f3354d0d Mon Sep 17 00:00:00 2001
From: Nitish <nitish@minio.io>
Date: Wed, 20 Sep 2017 08:12:14 +0530
Subject: [PATCH] Update mint to latest SDK releases and unify log output

Also includes following fixes:

* Add newline for minio-js logger for ease of reading
* Fix datafile creation as blocks greater than 34MB are not supported.
Use shred to create data files as it is faster and avoids the block
size issue with dd command
* Escape mc logs before printing as JSON

Fixes https://github.com/minio/mint/issues/35
---
 build/minio-go/install.sh          | 3 ++-
 build/minio-java/install.sh        | 2 +-
 build/minio-py/install.sh          | 2 +-
 create-data-files.sh               | 2 +-
 mint.sh                            | 6 +++---
 run/core/aws-sdk-php/run.sh        | 2 +-
 run/core/aws-sdk-ruby/run.sh       | 2 +-
 run/core/awscli/run.sh             | 2 +-
 run/core/mc/run.sh                 | 2 +-
 run/core/mc/test.sh                | 9 ++++++---
 run/core/minio-go/run.sh           | 2 +-
 run/core/minio-java/run.sh         | 5 +++--
 run/core/minio-js/minioreporter.js | 4 ++--
 run/core/minio-js/run.sh           | 2 +-
 run/core/minio-py/run.sh           | 2 +-
 15 files changed, 26 insertions(+), 21 deletions(-)

diff --git a/build/minio-go/install.sh b/build/minio-go/install.sh
index 0d1932af..7b98e4a5 100755
--- a/build/minio-go/install.sh
+++ b/build/minio-go/install.sh
@@ -15,10 +15,11 @@
 #  limitations under the License.
 #
 
-MINIO_GO_VERSION="v3.0.2"
+MINIO_GO_VERSION="3.0.3"
 
 test_run_dir="$MINT_RUN_CORE_DIR/minio-go"
 go get -u github.com/sirupsen/logrus/...
+go get -u github.com/dustin/go-humanize/...
 go get -u github.com/minio/minio-go/...
 (cd "$GOPATH/src/github.com/minio/minio-go" && git checkout --quiet "tags/$MINIO_GO_VERSION")
 CGO_ENABLED=0 go build -o "$test_run_dir/minio-go" "$GOPATH/src/github.com/minio/minio-go/functional_tests.go"
diff --git a/build/minio-java/install.sh b/build/minio-java/install.sh
index 6451b5ab..f8455d65 100755
--- a/build/minio-java/install.sh
+++ b/build/minio-java/install.sh
@@ -15,7 +15,7 @@
 #  limitations under the License.
 #
 
-MINIO_JAVA_VERSION="3.0.6"
+MINIO_JAVA_VERSION="3.0.7"
 
 test_run_dir="$MINT_RUN_CORE_DIR/minio-java"
 git clone --quiet https://github.com/minio/minio-java.git "$test_run_dir/minio-java.git"
diff --git a/build/minio-py/install.sh b/build/minio-py/install.sh
index 1985cf89..dee0b0a3 100755
--- a/build/minio-py/install.sh
+++ b/build/minio-py/install.sh
@@ -15,7 +15,7 @@
 #  limitations under the License.
 #
 
-MINIO_PY_VERSION="2.2.4"
+MINIO_PY_VERSION="2.2.5"
 
 test_run_dir="$MINT_RUN_CORE_DIR/minio-py"
 pip3 install --user faker
diff --git a/create-data-files.sh b/create-data-files.sh
index 7a3a90e9..ca912502 100755
--- a/create-data-files.sh
+++ b/create-data-files.sh
@@ -32,7 +32,7 @@ data_file_map["datafile-65-MB"]="65M"
 mkdir -p "$MINT_DATA_DIR"
 for filename in "${!data_file_map[@]}"; do
     echo "creating $MINT_DATA_DIR/$filename"
-    if ! dd if=/dev/urandom of="$MINT_DATA_DIR/$filename" bs="${data_file_map[$filename]}" count=1 >/dev/null 2>&1; then
+    if ! shred -s "${data_file_map[$filename]}" -> "$MINT_DATA_DIR/$filename" 2>&1; then
         echo "unable to create data file $MINT_DATA_DIR/$filename"
         exit 1
     fi
diff --git a/mint.sh b/mint.sh
index c5ab7fae..01c124a3 100755
--- a/mint.sh
+++ b/mint.sh
@@ -30,11 +30,11 @@ if [ -z "$SERVER_ENDPOINT" ]; then
 fi
 
 ROOT_DIR="$PWD"
-ERROR_LOG_FILE="error.log"
-OUTPUT_LOG_FILE="output.log"
 TESTS_DIR="$ROOT_DIR/run/core"
 
 BASE_LOG_DIR="$ROOT_DIR/log"
+LOG_FILE="log.json"
+ERROR_FILE="error.log"
 mkdir -p "$BASE_LOG_DIR"
 
 function humanize_time()
@@ -65,7 +65,7 @@ function run_test()
 
     mkdir -p "$BASE_LOG_DIR/$sdk_name"
 
-    (cd "$sdk_dir" && ./run.sh "$BASE_LOG_DIR/$sdk_name/$OUTPUT_LOG_FILE" "$BASE_LOG_DIR/$sdk_name/$ERROR_LOG_FILE")
+    (cd "$sdk_dir" && ./run.sh "$BASE_LOG_DIR/$LOG_FILE" "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE")
     rv=$?
     end=$(date +%s)
     duration=$(humanize_time $(( end - start )))
diff --git a/run/core/aws-sdk-php/run.sh b/run/core/aws-sdk-php/run.sh
index f190a25f..3a5c0b89 100755
--- a/run/core/aws-sdk-php/run.sh
+++ b/run/core/aws-sdk-php/run.sh
@@ -25,4 +25,4 @@ output_log_file="$1"
 error_log_file="$2"
 
 # run tests
-php ./quick-tests.php 1>"$output_log_file" 2>"$error_log_file"
+php ./quick-tests.php 1>>"$output_log_file" 2>"$error_log_file"
diff --git a/run/core/aws-sdk-ruby/run.sh b/run/core/aws-sdk-ruby/run.sh
index 82a8b55e..73e647b7 100755
--- a/run/core/aws-sdk-ruby/run.sh
+++ b/run/core/aws-sdk-ruby/run.sh
@@ -26,4 +26,4 @@ error_log_file="$2"
 
 # run tests
 chmod a+x aws-stub-tests.rb
-ruby aws-stub-tests.rb 1>"$output_log_file" 2>"$error_log_file"
+ruby aws-stub-tests.rb 1>>"$output_log_file" 2>"$error_log_file"
diff --git a/run/core/awscli/run.sh b/run/core/awscli/run.sh
index e2cafecb..613acf1f 100755
--- a/run/core/awscli/run.sh
+++ b/run/core/awscli/run.sh
@@ -34,4 +34,4 @@ endpoint="http://$SERVER_ENDPOINT"
 if [ "$ENABLE_HTTPS" -eq 1 ]; then
     endpoint="https://$SERVER_ENDPOINT"
 fi
-./test.sh "$endpoint"  1>"$output_log_file" 2>"$error_log_file"
\ No newline at end of file
+./test.sh "$endpoint"  1>>"$output_log_file" 2>"$error_log_file"
diff --git a/run/core/mc/run.sh b/run/core/mc/run.sh
index 769e881d..d442afeb 100755
--- a/run/core/mc/run.sh
+++ b/run/core/mc/run.sh
@@ -31,4 +31,4 @@ if [ "$ENABLE_HTTPS" -eq 1 ]; then
 fi
 
 ./mc --quiet config host add target "$endpoint" "$ACCESS_KEY" "$SECRET_KEY" >/dev/null 2>&1
-./test.sh 1>"$output_log_file" 2>"$error_log_file"
+./test.sh 1>>"$output_log_file" 2>"$error_log_file"
diff --git a/run/core/mc/test.sh b/run/core/mc/test.sh
index 21152d89..9d216223 100755
--- a/run/core/mc/test.sh
+++ b/run/core/mc/test.sh
@@ -31,15 +31,18 @@ function get_duration() {
 }
 
 function log_success() {
-    printf '{"name": "mc", "duration": "%d", "function": "%s", "status": "PASS"}\n' "$1" "$2"
+    function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2")
+    printf '{"name": "mc", "duration": "%d", "function": %s, "status": "PASS"}\n' "$1" "$function"
 }
 
 function log_failure() {
-    printf '{"name": "mc", "duration": "%d", "function": "%s", "status": "FAIL", "error": "%s"}\n' "$1" "$2" "$3"
+    function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2")
+    printf '{"name": "mc", "duration": "%d", "function": %s, "status": "FAIL", "error": %s}\n' "$1" "$function" "$3"
 }
 
 function log_alert() {
-    printf '{"name": "mc", "duration": "%d", "function": "%s", "status": "FAIL", "alert": "%d", error": "%s"}\n' "$1" "$2" "$3" "$4"
+    function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2")
+    printf '{"name": "mc", "duration": "%d", "function": %s, "status": "FAIL", "alert": "%d", error": "%s"}\n' "$1" "$function" "$3" "$4"
 }
 
 function make_bucket() {
diff --git a/run/core/minio-go/run.sh b/run/core/minio-go/run.sh
index 77779ee8..5ffa4b86 100755
--- a/run/core/minio-go/run.sh
+++ b/run/core/minio-go/run.sh
@@ -25,4 +25,4 @@ output_log_file="$1"
 error_log_file="$2"
 
 # run tests
-/mint/run/core/minio-go/minio-go 1>"$output_log_file" 2>"$error_log_file"
+/mint/run/core/minio-go/minio-go 1>>"$output_log_file" 2>"$error_log_file"
diff --git a/run/core/minio-java/run.sh b/run/core/minio-java/run.sh
index fe59eae1..f25dd786 100755
--- a/run/core/minio-java/run.sh
+++ b/run/core/minio-java/run.sh
@@ -15,7 +15,7 @@
 #  limitations under the License.
 #
 
-MINIO_JAVA_VERSION="3.0.6"
+MINIO_JAVA_VERSION="3.0.7"
 MINIO_JAVA_PATH="/mint/run/core/minio-java"
 
 # handle command line arguments
@@ -34,4 +34,5 @@ if [ "$ENABLE_HTTPS" -eq 1 ]; then
 fi
 
 java -cp "$MINIO_JAVA_PATH/minio-${MINIO_JAVA_VERSION}-all.jar:." FunctionalTest \
-     "$endpoint" "$ACCESS_KEY" "$SECRET_KEY" "$SERVER_REGION" 1>"$output_log_file" 2>"$error_log_file"
+"$endpoint" "$ACCESS_KEY" "$SECRET_KEY" "$SERVER_REGION" 1>>"$output_log_file" 2>"$error_log_file"
+     
\ No newline at end of file
diff --git a/run/core/minio-js/minioreporter.js b/run/core/minio-js/minioreporter.js
index 2ff3b29c..26292e8b 100644
--- a/run/core/minio-js/minioreporter.js
+++ b/run/core/minio-js/minioreporter.js
@@ -23,13 +23,13 @@ function minioreporter(runner) {
 
   runner.on('pass', function (test) {
     var res = test.title.split("_");
-    process.stdout.write(JSON.stringify({name: "minio-js", status: "PASS", function: res[0], args: res[1], alert: res[2], duration: test.duration}));
+    process.stdout.write(JSON.stringify({name: "minio-js", status: "PASS", function: res[0], args: res[1], alert: res[2], duration: test.duration}) + "\n");
   });
 
   runner.on('fail', function (test) {
      test.err = errorJSON (test.error)
      var res = test.title.split("_");
-    process.stdout.write(JSON.stringify({name: "minio-js", status: "FAIL", function: res[0], args: res[1], alert: res[2], duration: test.duration, error: test.err}));
+    process.stdout.write(JSON.stringify({name: "minio-js", status: "FAIL", function: res[0], args: res[1], alert: res[2], duration: test.duration, error: test.err}) + "\n");
   });
 
 }
diff --git a/run/core/minio-js/run.sh b/run/core/minio-js/run.sh
index 403d2904..c01a8fc6 100755
--- a/run/core/minio-js/run.sh
+++ b/run/core/minio-js/run.sh
@@ -25,4 +25,4 @@ output_log_file="$1"
 error_log_file="$2"
 
 # run tests
-./node_modules/mocha/bin/mocha -R minioreporter -b 1>"$output_log_file" 2>"$error_log_file"
+./node_modules/mocha/bin/mocha -R minioreporter -b 1>>"$output_log_file" 2>"$error_log_file"
diff --git a/run/core/minio-py/run.sh b/run/core/minio-py/run.sh
index 193e7684..7d15357b 100755
--- a/run/core/minio-py/run.sh
+++ b/run/core/minio-py/run.sh
@@ -25,4 +25,4 @@ output_log_file="$1"
 error_log_file="$2"
 
 # run tests
-python "/mint/run/core/minio-py/tests.py" 1>"$output_log_file" 2>"$error_log_file"
+python "/mint/run/core/minio-py/tests.py" 1>>"$output_log_file" 2>"$error_log_file"