From 37a6cd87380a89e50a65b284906dfdfe03e5a87a Mon Sep 17 00:00:00 2001 From: Praveen raj Mani Date: Thu, 28 Jun 2018 11:04:36 +0530 Subject: [PATCH] The awscli tests for virtual style paths (dnsmasq fix) (#278) - Add new mode to test virtual host style with awscli Mint awscli tests now run against both path style and virtual style if virtual style flag is set. By default only path style tests are run - The dnsmasq will now resolvethe wildcard entries also This will now facilitate the randomized wildcard bucket names in awscli tests (virtual style paths). Fixes #213 --- Dockerfile | 2 +- Dockerfile.dev | 2 +- README.md | 18 +++++++++++++++- mint.sh | 48 ++++++++++++++++++++++++++++++------------ run/core/awscli/run.sh | 16 +++++++++++++- 5 files changed, 68 insertions(+), 18 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3c2e644f..409d4952 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ ENV GOPATH /usr/local ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH -RUN apt-get --yes update && apt-get --yes upgrade && apt-get --yes --quiet install wget jq curl git && \ +RUN apt-get --yes update && apt-get --yes upgrade && apt-get --yes --quiet install wget jq curl git dnsmasq && \ git clone https://github.com/minio/mint.git /mint && \ cd /mint && /mint/release.sh diff --git a/Dockerfile.dev b/Dockerfile.dev index 7575743a..153638b2 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -12,7 +12,7 @@ ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH WORKDIR /mint -RUN apt-get --yes update && apt-get --yes upgrade && apt-get --yes --quiet install wget jq curl +RUN apt-get --yes update && apt-get --yes upgrade && apt-get --yes --quiet install wget jq curl dnsmasq ENV MINT_ROOT_DIR /mint ENV MINT_RUN_CORE_DIR $MINT_ROOT_DIR/run/core diff --git a/README.md b/README.md index b272761e..04cc48fc 100644 --- a/README.md +++ b/README.md @@ -38,12 +38,28 @@ Below environment variables are required to be passed to the docker container. S | Environment variable | Description | Example | |:--- |:--- |:--- | -| `SERVER_ENDPOINT` | Endpoint of Minio server in the format `HOST:PORT` | `play.minio.io:9000` | +| `SERVER_ENDPOINT` | Endpoint of Minio server in the format `HOST:PORT`; for virtual style `IP:PORT` | `play.minio.io:9000` | | `ACCESS_KEY` | Access key of access `SERVER_ENDPOINT` | `Q3AM3UQ867SPQQA43P2F` | | `SECRET_KEY` | Secret Key of access `SERVER_ENDPOINT` | `zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG` | | `ENABLE_HTTPS` | (Optional) Set `1` to indicate to use HTTPS to access `SERVER_ENDPOINT`. Defaults to `0` (HTTP) | `1` | | `MINT_MODE` | (Optional) Set mode indicating what category of tests to be run by values `core`, `full` or `worm`. Defaults to `core` | `full` | +| `DOMAIN` | (Optional) Value of MINIO_DOMAIN environment variable used in Minio server | `myminio.com` | +| `ENABLE_VIRTUAL_STYLE` | (Optional) Set `1` to indicate virtual style access . Defaults to `0` (Path style) | `1` | + +### Test virtual style access against Minio server + +To test Minio server virtual style access with Mint, follow these steps: + +- Set a domain in your Minio server using environment variable MINIO_DOMAIN. For example `export MINIO_DOMAIN=myminio.com`. +- Start Minio server. +- Execute Mint against Minio server (with `MINIO_DOMAIN` set to `myminio.com`) using this command +```sh +$ docker run -e "SERVER_ENDPOINT=192.168.86.133:9000" -e "DOMAIN=minio.com" \ + -e "ACCESS_KEY=minio" -e "SECRET_KEY=minio123" -e "ENABLE_HTTPS=0" \ + -e "ENABLE_VIRTUAL_STYLE=1" minio/mint +``` + ### Mint log format All test logs are stored in `/mint/log/log.json` as multiple JSON document. Below is the JSON format for every entry in the log file. diff --git a/mint.sh b/mint.sh index adca7070..75f51de6 100755 --- a/mint.sh +++ b/mint.sh @@ -20,6 +20,7 @@ MINT_DATA_DIR=${MINT_DATA_DIR:-/mint/data} MINT_MODE=${MINT_MODE:-core} SERVER_REGION=${SERVER_REGION:-us-east-1} ENABLE_HTTPS=${ENABLE_HTTPS:-0} +ENABLE_VIRTUAL_STYLE=${ENABLE_VIRTUAL_STYLE:-0} if [ -z "$SERVER_ENDPOINT" ]; then SERVER_ENDPOINT="play.minio.io:9000" @@ -28,6 +29,23 @@ if [ -z "$SERVER_ENDPOINT" ]; then ENABLE_HTTPS=1 fi +if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then + SERVER_IP="${SERVER_ENDPOINT%%:*}" + SERVER_PORT="${SERVER_ENDPOINT/*:/}" + # Check if SERVER_IP is actually IPv4 address + octets=("${SERVER_IP//./ }") + if [ "${#octets[@]}" -ne 4 ]; then + echo "$SERVER_IP must be an IP address" + exit 1 + fi + for octet in "${octets[@]}"; do + if [ "$octet" -lt 0 ] 2>/dev/null || [ "$octet" -gt 255 ] 2>/dev/null; then + echo "$SERVER_IP must be an IP address" + exit 1 + fi + done +fi + ROOT_DIR="$PWD" TESTS_DIR="$ROOT_DIR/run/core" @@ -84,7 +102,6 @@ function run_test() jq . <<<"$entry" fi fi - return $rv } @@ -93,23 +110,27 @@ function main() export MINT_DATA_DIR export MINT_MODE export SERVER_ENDPOINT + export SERVER_IP + export SERVER_PORT + export ACCESS_KEY export SECRET_KEY export ENABLE_HTTPS export SERVER_REGION - + export ENABLE_VIRTUAL_STYLE + echo "Running with" - echo "SERVER_ENDPOINT: $SERVER_ENDPOINT" - echo "ACCESS_KEY: $ACCESS_KEY" - echo "SECRET_KEY: ***REDACTED***" - echo "ENABLE_HTTPS: $ENABLE_HTTPS" - echo "SERVER_REGION: $SERVER_REGION" - echo "MINT_DATA_DIR: $MINT_DATA_DIR" - echo "MINT_MODE: $MINT_MODE" + echo "SERVER_ENDPOINT: $SERVER_ENDPOINT" + echo "ACCESS_KEY: $ACCESS_KEY" + echo "SECRET_KEY: ***REDACTED***" + echo "ENABLE_HTTPS: $ENABLE_HTTPS" + echo "SERVER_REGION: $SERVER_REGION" + echo "MINT_DATA_DIR: $MINT_DATA_DIR" + echo "MINT_MODE: $MINT_MODE" + echo "ENABLE_VIRTUAL_STYLE: $ENABLE_VIRTUAL_STYLE" echo - echo "To get logs, run 'docker cp ${CONTAINER_ID}:/mint/log /tmp/mint-logs'" - + declare -a run_list if [ "$MINT_MODE" == "worm" ]; then if [ "$#" -gt 1 ]; then @@ -135,7 +156,7 @@ function main() run_list=( "${run_list[@]}" "$TESTS_DIR/$sdk" ) done fi - + count="${#run_list[@]}" i=0 for sdk_dir in "${run_list[@]}"; do @@ -151,7 +172,7 @@ function main() break fi done - + ## Report when all tests in run_list are run if [ $i -eq "$count" ]; then echo -e "\nAll tests ran successfully" @@ -160,5 +181,4 @@ function main() exit 1 fi } - main "$@" diff --git a/run/core/awscli/run.sh b/run/core/awscli/run.sh index 613acf1f..7ce73ee1 100755 --- a/run/core/awscli/run.sh +++ b/run/core/awscli/run.sh @@ -29,9 +29,23 @@ aws configure set aws_access_key_id "$ACCESS_KEY" aws configure set aws_secret_access_key "$SECRET_KEY" aws configure set default.region "$SERVER_REGION" -# run tests +# run tests for virtual style if provided +if [ "$ENABLE_VIRTUAL_STYLE" -eq 1 ]; then + # Setup endpoint scheme + endpoint="http://$DOMAIN:$SERVER_PORT" + if [ "$ENABLE_HTTPS" -eq 1 ]; then + endpoint="https://$DOMAIN:$SERVER_PORT" + fi + dnsmasq --address="/$DOMAIN/$SERVER_IP" --user=root + echo -e "nameserver 127.0.0.1\n$(cat /etc/resolv.conf)" > /etc/resolv.conf + aws configure set default.s3.addressing_style virtual + ./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file" + aws configure set default.s3.addressing_style path +fi + endpoint="http://$SERVER_ENDPOINT" if [ "$ENABLE_HTTPS" -eq 1 ]; then endpoint="https://$SERVER_ENDPOINT" fi +# run path style tests ./test.sh "$endpoint" 1>>"$output_log_file" 2>"$error_log_file"