Update mint to latest SDK releases and unify log output

Also includes following fixes:

* Add newline for minio-js logger for ease of reading
* Fix datafile creation as blocks greater than 34MB are not supported.
Use shred to create data files as it is faster and avoids the block
size issue with dd command
* Escape mc logs before printing as JSON

Fixes #35

build/minio-go/install.sh

@@ -15,10 +15,11 @@
 #  limitations under the License.
 #
 
-MINIO_GO_VERSION="v3.0.2"
+MINIO_GO_VERSION="3.0.3"
 
 test_run_dir="$MINT_RUN_CORE_DIR/minio-go"
 go get -u github.com/sirupsen/logrus/...
+go get -u github.com/dustin/go-humanize/...
 go get -u github.com/minio/minio-go/...
 (cd "$GOPATH/src/github.com/minio/minio-go" && git checkout --quiet "tags/$MINIO_GO_VERSION")
 CGO_ENABLED=0 go build -o "$test_run_dir/minio-go" "$GOPATH/src/github.com/minio/minio-go/functional_tests.go"

build/minio-java/install.sh

@@ -15,7 +15,7 @@
 #  limitations under the License.
 #
 
-MINIO_JAVA_VERSION="3.0.6"
+MINIO_JAVA_VERSION="3.0.7"
 
 test_run_dir="$MINT_RUN_CORE_DIR/minio-java"
 git clone --quiet https://github.com/minio/minio-java.git "$test_run_dir/minio-java.git"

build/minio-py/install.sh

@@ -15,7 +15,7 @@
 #  limitations under the License.
 #
 
-MINIO_PY_VERSION="2.2.4"
+MINIO_PY_VERSION="2.2.5"
 
 test_run_dir="$MINT_RUN_CORE_DIR/minio-py"
 pip3 install --user faker

create-data-files.sh

@@ -32,7 +32,7 @@ data_file_map["datafile-65-MB"]="65M"
 mkdir -p "$MINT_DATA_DIR"
 for filename in "${!data_file_map[@]}"; do
     echo "creating $MINT_DATA_DIR/$filename"
-    if ! dd if=/dev/urandom of="$MINT_DATA_DIR/$filename" bs="${data_file_map[$filename]}" count=1 >/dev/null 2>&1; then
+    if ! shred -s "${data_file_map[$filename]}" -> "$MINT_DATA_DIR/$filename" 2>&1; then
         echo "unable to create data file $MINT_DATA_DIR/$filename"
         exit 1
     fi

mint.sh

@@ -30,11 +30,11 @@ if [ -z "$SERVER_ENDPOINT" ]; then
 fi
 
 ROOT_DIR="$PWD"
-ERROR_LOG_FILE="error.log"
-OUTPUT_LOG_FILE="output.log"
 TESTS_DIR="$ROOT_DIR/run/core"
 
 BASE_LOG_DIR="$ROOT_DIR/log"
+LOG_FILE="log.json"
+ERROR_FILE="error.log"
 mkdir -p "$BASE_LOG_DIR"
 
 function humanize_time()
@@ -65,7 +65,7 @@ function run_test()
 
     mkdir -p "$BASE_LOG_DIR/$sdk_name"
 
-    (cd "$sdk_dir" && ./run.sh "$BASE_LOG_DIR/$sdk_name/$OUTPUT_LOG_FILE" "$BASE_LOG_DIR/$sdk_name/$ERROR_LOG_FILE")
+    (cd "$sdk_dir" && ./run.sh "$BASE_LOG_DIR/$LOG_FILE" "$BASE_LOG_DIR/$sdk_name/$ERROR_FILE")
     rv=$?
     end=$(date +%s)
     duration=$(humanize_time $(( end - start )))

run/core/aws-sdk-php/run.sh

@@ -25,4 +25,4 @@ output_log_file="$1"
 error_log_file="$2"
 
 # run tests
-php ./quick-tests.php 1>"$output_log_file" 2>"$error_log_file"
+php ./quick-tests.php 1>>"$output_log_file" 2>"$error_log_file"

run/core/aws-sdk-ruby/run.sh

@@ -26,4 +26,4 @@ error_log_file="$2"
 
 # run tests
 chmod a+x aws-stub-tests.rb
-ruby aws-stub-tests.rb 1>"$output_log_file" 2>"$error_log_file"
+ruby aws-stub-tests.rb 1>>"$output_log_file" 2>"$error_log_file"

run/core/awscli/run.sh

@@ -34,4 +34,4 @@ endpoint="http://$SERVER_ENDPOINT"
 if [ "$ENABLE_HTTPS" -eq 1 ]; then
     endpoint="https://$SERVER_ENDPOINT"
 fi
-./test.sh "$endpoint"  1>"$output_log_file" 2>"$error_log_file"
+./test.sh "$endpoint"  1>>"$output_log_file" 2>"$error_log_file"

run/core/mc/run.sh

@@ -31,4 +31,4 @@ if [ "$ENABLE_HTTPS" -eq 1 ]; then
 fi
 
 ./mc --quiet config host add target "$endpoint" "$ACCESS_KEY" "$SECRET_KEY" >/dev/null 2>&1
-./test.sh 1>"$output_log_file" 2>"$error_log_file"
+./test.sh 1>>"$output_log_file" 2>"$error_log_file"

run/core/mc/test.sh

@@ -31,15 +31,18 @@ function get_duration() {
 }
 
 function log_success() {
-    printf '{"name": "mc", "duration": "%d", "function": "%s", "status": "PASS"}\n' "$1" "$2"
+    function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2")
+    printf '{"name": "mc", "duration": "%d", "function": %s, "status": "PASS"}\n' "$1" "$function"
 }
 
 function log_failure() {
-    printf '{"name": "mc", "duration": "%d", "function": "%s", "status": "FAIL", "error": "%s"}\n' "$1" "$2" "$3"
+    function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2")
+    printf '{"name": "mc", "duration": "%d", "function": %s, "status": "FAIL", "error": %s}\n' "$1" "$function" "$3"
 }
 
 function log_alert() {
-    printf '{"name": "mc", "duration": "%d", "function": "%s", "status": "FAIL", "alert": "%d", error": "%s"}\n' "$1" "$2" "$3" "$4"
+    function=$(python -c 'import sys,json; print(json.dumps(sys.stdin.read()))' <<<"$2")
+    printf '{"name": "mc", "duration": "%d", "function": %s, "status": "FAIL", "alert": "%d", error": "%s"}\n' "$1" "$function" "$3" "$4"
 }
 
 function make_bucket() {

run/core/minio-go/run.sh

@@ -25,4 +25,4 @@ output_log_file="$1"
 error_log_file="$2"
 
 # run tests
-/mint/run/core/minio-go/minio-go 1>"$output_log_file" 2>"$error_log_file"
+/mint/run/core/minio-go/minio-go 1>>"$output_log_file" 2>"$error_log_file"

run/core/minio-java/run.sh

@@ -15,7 +15,7 @@
 #  limitations under the License.
 #
 
-MINIO_JAVA_VERSION="3.0.6"
+MINIO_JAVA_VERSION="3.0.7"
 MINIO_JAVA_PATH="/mint/run/core/minio-java"
 
 # handle command line arguments
@@ -34,4 +34,5 @@ if [ "$ENABLE_HTTPS" -eq 1 ]; then
 fi
 
 java -cp "$MINIO_JAVA_PATH/minio-${MINIO_JAVA_VERSION}-all.jar:." FunctionalTest \
-     "$endpoint" "$ACCESS_KEY" "$SECRET_KEY" "$SERVER_REGION" 1>"$output_log_file" 2>"$error_log_file"
+"$endpoint" "$ACCESS_KEY" "$SECRET_KEY" "$SERVER_REGION" 1>>"$output_log_file" 2>"$error_log_file"
+

run/core/minio-js/minioreporter.js

@@ -23,13 +23,13 @@ function minioreporter(runner) {
 
   runner.on('pass', function (test) {
     var res = test.title.split("_");
-    process.stdout.write(JSON.stringify({name: "minio-js", status: "PASS", function: res[0], args: res[1], alert: res[2], duration: test.duration}));
+    process.stdout.write(JSON.stringify({name: "minio-js", status: "PASS", function: res[0], args: res[1], alert: res[2], duration: test.duration}) + "\n");
   });
 
   runner.on('fail', function (test) {
      test.err = errorJSON (test.error)
      var res = test.title.split("_");
-    process.stdout.write(JSON.stringify({name: "minio-js", status: "FAIL", function: res[0], args: res[1], alert: res[2], duration: test.duration, error: test.err}));
+    process.stdout.write(JSON.stringify({name: "minio-js", status: "FAIL", function: res[0], args: res[1], alert: res[2], duration: test.duration, error: test.err}) + "\n");
   });
 
 }

run/core/minio-js/run.sh

@@ -25,4 +25,4 @@ output_log_file="$1"
 error_log_file="$2"
 
 # run tests
-./node_modules/mocha/bin/mocha -R minioreporter -b 1>"$output_log_file" 2>"$error_log_file"
+./node_modules/mocha/bin/mocha -R minioreporter -b 1>>"$output_log_file" 2>"$error_log_file"

run/core/minio-py/run.sh

@@ -25,4 +25,4 @@ output_log_file="$1"
 error_log_file="$2"
 
 # run tests
-python "/mint/run/core/minio-py/tests.py" 1>"$output_log_file" 2>"$error_log_file"
+python "/mint/run/core/minio-py/tests.py" 1>>"$output_log_file" 2>"$error_log_file"