From 4b0876110ce77e5be84ea051cad9bccf53642acb Mon Sep 17 00:00:00 2001 From: Yingrong Zhao Date: Wed, 11 Dec 2019 11:39:06 -0500 Subject: [PATCH] delete ul-cfg.sh and add support for both mac and linux Change-Id: I60451af04e4ff33c466a5e5a3e5b4c69c60b7769 --- scripts/test-sim-versions.sh | 13 ++- scripts/test-versions-custom-ul-cfg.sh | 154 ------------------------- scripts/test-versions.sh | 36 +++--- 3 files changed, 30 insertions(+), 173 deletions(-) delete mode 100755 scripts/test-versions-custom-ul-cfg.sh diff --git a/scripts/test-sim-versions.sh b/scripts/test-sim-versions.sh index ac7b8d64fe7d..355478607e3c 100755 --- a/scripts/test-sim-versions.sh +++ b/scripts/test-sim-versions.sh @@ -24,13 +24,13 @@ populates_sno_versions(){ # in stage 1: satellite and storagenode use latest release version, uplink uses all highest point release from all major releases starting from v0.15 # in stage 2: satellite core uses latest release version and satellite api uses master. Storage nodes are splited into half on latest release version and half on master. Uplink uses the all versions from stage 1 plus master current_release_version=$(git describe --tags `git rev-list --tags --max-count=1`) -majorReleaseTags=$(git tag -l --sort -version:refname | sort -k2,2 -t'.' --unique | grep -e "^v0\.\(1[5-9]\)\|2[2-9]") +major_release_tags=$(git tag -l --sort -version:refname | sort -k2,2 -t'.' --unique | grep -e "^v0\.\(1[5-9]\)\|2[2-9]") stage1_sat_version=$current_release_version -stage1_uplink_versions=$majorReleaseTags +stage1_uplink_versions=$major_release_tags stage1_storagenode_versions=$(populates_sno_versions $current_release_version 10) # TODO separate satellite version into satellite api version and satellite core verion stage2_sat_version="master" -stage2_uplink_versions=$majorReleaseTags\ "master" +stage2_uplink_versions=$major_release_tags\ "master" stage2_storagenode_versions=$(populates_sno_versions $current_release_version 5)\ $(populates_sno_versions "master" 5) echo "stage1_sat_version" $stage1_sat_version @@ -54,7 +54,12 @@ replace_in_file(){ local src="$1" local dest="$2" local path=$3 - sed -i "s#${src}#${dest}#g" "${path}" + case "$OSTYPE" in + darwin*) + sed -i '' "s#${src}#${dest}#g" "${path}" ;; + *) + sed -i "s#${src}#${dest}#g" "${path}" ;; + esac } diff --git a/scripts/test-versions-custom-ul-cfg.sh b/scripts/test-versions-custom-ul-cfg.sh deleted file mode 100755 index c0d8916e34a7..000000000000 --- a/scripts/test-versions-custom-ul-cfg.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env bash -set -ueo pipefail - -main_cfg_dir=$1 -command=$2 - -bucket="bucket-123" -test_files_dir="${main_cfg_dir}/testfiles" -stage1_dst_dir="${main_cfg_dir}/stage1" -stage2_dst_dir="${main_cfg_dir}/stage2" - -setup(){ - mkdir -p "$test_files_dir" "$stage1_dst_dir" "$stage2_dst_dir" - random_bytes_file () { - size=$1 - output=$2 - head -c $size $output - } - random_bytes_file "2048" "$test_files_dir/small-upload-testfile" # create 2kb file of random bytes (inline) - random_bytes_file "5242880" "$test_files_dir/big-upload-testfile" # create 5mb file of random bytes (remote) - random_bytes_file "134217728" "$test_files_dir/multisegment-upload-testfile" # create 128mb file of random bytes (remote) - - echo "setup test successfully" -} - -echo "Begin test-versions.sh, storj-sim config directory:" ${main_cfg_dir} - -echo "which storj-sim: $(which storj-sim)" -# shasum $(which storj-sim) - -echo -e "\nConfig directory for uplink:" -mkdir ${main_cfg_dir}/uplink -echo "${main_cfg_dir}/uplink" -echo "which uplink: $(which uplink)" -echo "Shasum for uplink:" -shasum $(which uplink) - -api_key=$(storj-sim --config-dir=$main_cfg_dir network env GATEWAY_0_API_KEY) -sat_addr=$(storj-sim --config-dir=$main_cfg_dir network env SATELLITE_0_ADDR) -uplink setup --non-interactive --api-key="$api_key" --satellite-addr="$sat_addr" --config-dir="${main_cfg_dir}/uplink" -# uplink version --config-dir "${main_cfg_dir}/gateway/0/" - -echo -e "\nConfig directory for satellite:" -echo "${main_cfg_dir}/satellite/0" -# storj-sim network env --config-dir "${main_cfg_dir}" SATELLITE_0_DIR -echo "Shasum for satellite:" -shasum ${main_cfg_dir}/satellite/0/satellite - -echo -e "\nStoragenode config directories:" -for i in {0..9} -do - echo -e "\nConfig directory for sn ${i}:" - echo "${main_cfg_dir}/storagenode/${i}" - # storj-sim network env --config-dir "${main_cfg_dir}" STORAGENODE_${i}_DIR - echo "Shasum for sn ${i} binary:" - shasum ${main_cfg_dir}/storagenode/${i}/storagenode -done - -if [[ "$command" == "upload" ]]; then - uplink_version=$3 - setup - bucket_name=${bucket}-${uplink_version} - download_dst_dir=${stage1_dst_dir}/${uplink_version} - mkdir -p "$download_dst_dir" - - uplink mb "sj://$bucket_name/" --config-dir="${main_cfg_dir}/uplink" - - uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/small-upload-testfile" "sj://$bucket_name/" - uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/big-upload-testfile" "sj://$bucket_name/" - uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/multisegment-upload-testfile" "sj://$bucket_name/" - - uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}" - uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}" - uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}" - - if cmp "${test_files_dir}/small-upload-testfile" "${download_dst_dir}/small-upload-testfile" - then - echo "upload test on release tag: small upload testfile matches uploaded file" - else - echo "upload test on release tag: small upload testfile does not match uploaded file" - exit 1 - fi - - if cmp "${test_files_dir}/big-upload-testfile" "${download_dst_dir}/big-upload-testfile" - then - echo "upload test on release tag: big upload testfile matches uploaded file" - else - echo "upload test on release tag: big upload testfile does not match uploaded file" - exit 1 - fi - - if cmp "${test_files_dir}/multisegment-upload-testfile" "${download_dst_dir}/multisegment-upload-testfile" - then - echo "upload test on release tag: multisegment upload testfile matches uploaded file" - else - echo "upload test on release tag: multisegment upload testfile does not match uploaded file" - exit 1 - fi - -fi - -if [[ "$command" == "download" ]]; then - existing_bucket_name_suffixes=$3 - - for suffix in ${existing_bucket_name_suffixes}; do - bucket_name=${bucket}-${suffix} - original_dst_dir=${stage1_dst_dir}/${suffix} - download_dst_dir=${stage2_dst_dir}/${suffix} - mkdir -p "$download_dst_dir" - - echo "bucket name: ${bucket_name}" - echo "download folder name: ${download_dst_dir}" - uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}" - uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}" - uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}" - - if cmp "${original_dst_dir}/small-upload-testfile" "${download_dst_dir}/small-upload-testfile" - then - echo "download test on current branch: small upload testfile matches uploaded file" - else - echo "download test on current branch: small upload testfile does not match uploaded file" - exit 1 - fi - - if cmp "${original_dst_dir}/big-upload-testfile" "${download_dst_dir}/big-upload-testfile" - then - echo "download test on current branch: big upload testfile matches uploaded file" - else - echo "download test on current branch: big upload testfile does not match uploaded file" - exit 1 - fi - - if cmp "${original_dst_dir}/multisegment-upload-testfile" "${download_dst_dir}/multisegment-upload-testfile" - then - echo "download test on current branch: multisegment upload testfile matches uploaded file" - else - echo "download test on current branch: multisegment upload testfile does not match uploaded file" - exit 1 - fi - done -fi - -if [[ "$command" == "cleanup" ]]; then - uplink_versions=$3 - for ul_version in ${uplink_versions}; do - bucket_name=${bucket}-${ul_version} - uplink rm --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name/small-upload-testfile" - uplink rm --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name/big-upload-testfile" - uplink rm --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name/multisegment-upload-testfile" - uplink rb --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name" - done -fi - -echo "Done with test-versions.sh" diff --git a/scripts/test-versions.sh b/scripts/test-versions.sh index 3f3c28714fdf..dff6a56c1d53 100755 --- a/scripts/test-versions.sh +++ b/scripts/test-versions.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash + set -ueo pipefail main_cfg_dir=$1 @@ -29,10 +30,15 @@ echo "which storj-sim: $(which storj-sim)" # shasum $(which storj-sim) echo -e "\nConfig directory for uplink:" -echo "${main_cfg_dir}/gateway/0" +mkdir ${main_cfg_dir}/uplink +echo "${main_cfg_dir}/uplink" echo "which uplink: $(which uplink)" echo "Shasum for uplink:" shasum $(which uplink) + +api_key=$(storj-sim --config-dir=$main_cfg_dir network env GATEWAY_0_API_KEY) +sat_addr=$(storj-sim --config-dir=$main_cfg_dir network env SATELLITE_0_ADDR) +uplink setup --non-interactive --api-key="$api_key" --satellite-addr="$sat_addr" --config-dir="${main_cfg_dir}/uplink" # uplink version --config-dir "${main_cfg_dir}/gateway/0/" echo -e "\nConfig directory for satellite:" @@ -58,15 +64,15 @@ if [[ "$command" == "upload" ]]; then download_dst_dir=${stage1_dst_dir}/${uplink_version} mkdir -p "$download_dst_dir" - uplink --config-dir "${main_cfg_dir}/gateway/0" mb "sj://$bucket_name/" + uplink mb "sj://$bucket_name/" --config-dir="${main_cfg_dir}/uplink" - uplink --config-dir "${main_cfg_dir}/gateway/0" cp --progress=false "${test_files_dir}/small-upload-testfile" "sj://$bucket_name/" - uplink --config-dir "${main_cfg_dir}/gateway/0" cp --progress=false "${test_files_dir}/big-upload-testfile" "sj://$bucket_name/" - uplink --config-dir "${main_cfg_dir}/gateway/0" cp --progress=false "${test_files_dir}/multisegment-upload-testfile" "sj://$bucket_name/" + uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/small-upload-testfile" "sj://$bucket_name/" + uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/big-upload-testfile" "sj://$bucket_name/" + uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "${test_files_dir}/multisegment-upload-testfile" "sj://$bucket_name/" - uplink --config-dir "${main_cfg_dir}/gateway/0" cp --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}" - uplink --config-dir "${main_cfg_dir}/gateway/0" cp --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}" - uplink --config-dir "${main_cfg_dir}/gateway/0" cp --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}" + uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}" + uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}" + uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}" if cmp "${test_files_dir}/small-upload-testfile" "${download_dst_dir}/small-upload-testfile" then @@ -105,9 +111,9 @@ if [[ "$command" == "download" ]]; then echo "bucket name: ${bucket_name}" echo "download folder name: ${download_dst_dir}" - uplink --config-dir "${main_cfg_dir}/gateway/0" cp --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}" - uplink --config-dir "${main_cfg_dir}/gateway/0" cp --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}" - uplink --config-dir "${main_cfg_dir}/gateway/0" cp --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}" + uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/small-upload-testfile" "${download_dst_dir}" + uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/big-upload-testfile" "${download_dst_dir}" + uplink cp --config-dir="${main_cfg_dir}/uplink" --progress=false "sj://$bucket_name/multisegment-upload-testfile" "${download_dst_dir}" if cmp "${original_dst_dir}/small-upload-testfile" "${download_dst_dir}/small-upload-testfile" then @@ -139,10 +145,10 @@ if [[ "$command" == "cleanup" ]]; then uplink_versions=$3 for ul_version in ${uplink_versions}; do bucket_name=${bucket}-${ul_version} - uplink --config-dir "${main_cfg_dir}/gateway/0" rm "sj://$bucket_name/small-upload-testfile" - uplink --config-dir "${main_cfg_dir}/gateway/0" rm "sj://$bucket_name/big-upload-testfile" - uplink --config-dir "${main_cfg_dir}/gateway/0" rm "sj://$bucket_name/multisegment-upload-testfile" - uplink --config-dir "${main_cfg_dir}/gateway/0" rb "sj://$bucket_name" + uplink rm --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name/small-upload-testfile" + uplink rm --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name/big-upload-testfile" + uplink rm --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name/multisegment-upload-testfile" + uplink rb --config-dir="${main_cfg_dir}/uplink" "sj://$bucket_name" done fi