Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improved integration tests #575

Merged
merged 17 commits into from
May 20, 2024
Merged

Improved integration tests #575

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
e78f379
Improved tests
VukW Apr 9, 2024
10d4452
debug
VukW Apr 9, 2024
b0b1c8f
Debug
VukW Apr 9, 2024
fed9395
debug
VukW Apr 9, 2024
79139c9
debug
VukW Apr 9, 2024
e7df3b9
debug
VukW Apr 9, 2024
0f813b9
Fix: reodered folder creation and clean on -f
VukW Apr 9, 2024
7fc2750
import os breaks pytest
VukW Apr 9, 2024
1f144ba
Renamed `ev` to `print_eval`
VukW Apr 9, 2024
3903beb
Instead of real storage moving just set up config properly
VukW May 6, 2024
631c910
Merge remote-tracking branch 'origin/main' into integration-tests-eas…
VukW May 6, 2024
42a01ff
bugfix
VukW May 6, 2024
783f1cd
bugfix
VukW May 7, 2024
6f12b1e
bugfix
VukW May 7, 2024
eda3d93
bugfix
VukW May 7, 2024
8ea2c40
Covered multisource tests also
VukW May 7, 2024
8426e7a
Merge branch 'main' into integration-tests-ease-debug
hasan7n May 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 31 additions & 28 deletions cli/cli_chestxray_tutorial_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,42 +13,45 @@ wget -P $DIRECTORY https://storage.googleapis.com/medperf-storage/chestxray_tuto
tar -xzvf $DIRECTORY/sample_raw_data.tar.gz -C $DIRECTORY
chmod a+w $DIRECTORY/sample_raw_data

##########################################################


echo "=========================================="
echo "Creating test profiles for each user"
echo "=========================================="
medperf profile activate local
print_eval medperf profile activate local
checkFailed "local profile creation failed"

medperf profile create -n testbenchmark
print_eval medperf profile create -n testbenchmark
checkFailed "testbenchmark profile creation failed"
medperf profile create -n testdata
print_eval medperf profile create -n testdata
checkFailed "testdata profile creation failed"

echo "=========================================="
echo "Login each user"
echo "=========================================="
medperf profile activate testbenchmark
print_eval medperf profile activate testbenchmark
checkFailed "testbenchmark profile activation failed"

medperf auth login -e $BENCHMARKOWNER
print_eval medperf auth login -e $BENCHMARKOWNER
checkFailed "testbenchmark login failed"

medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"

medperf auth login -e $DATAOWNER
print_eval medperf auth login -e $DATAOWNER
checkFailed "testdata login failed"

echo "====================================="
echo "Activate dataowner profile"
echo "====================================="
medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"
echo "\n"
echo "====================================="
echo "Registering dataset with medperf"
echo "====================================="
medperf dataset submit -b 1 -d $DIRECTORY/sample_raw_data/images -l $DIRECTORY/sample_raw_data/labels --name="nih_chestxray" --description="sample dataset" --location="mock location" -y
print_eval "medperf dataset submit -b 1 -d $DIRECTORY/sample_raw_data/images -l $DIRECTORY/sample_raw_data/labels --name='nih_chestxray' --description='sample dataset' --location='mock location' -y"
checkFailed "Data registration step failed"

echo "\n"
Expand All @@ -57,86 +60,86 @@ echo "Dataset UID: $DSET_UID"
echo "====================================="
echo "Running data preparation step"
echo "====================================="
medperf dataset prepare -d $DSET_UID
print_eval medperf dataset prepare -d $DSET_UID
checkFailed "Data preparation step failed"

echo "\n"

echo "====================================="
echo "Running data set operational step"
echo "====================================="
medperf dataset set_operational -d $DSET_UID -y
print_eval medperf dataset set_operational -d $DSET_UID -y
checkFailed "Data set operational step failed"

echo "====================================="
echo "Creating dataset benchmark association"
echo "====================================="
medperf dataset associate -d $DSET_UID -b 1 -y
print_eval medperf dataset associate -d $DSET_UID -b 1 -y
checkFailed "Data association step failed"

echo "====================================="
echo ""Activate benchmarkowner profile""
echo "====================================="
# Log in as the benchmark owner
medperf profile activate testbenchmark
print_eval medperf profile activate testbenchmark
checkFailed "testbenchmark profile activation failed"
# Get association information
ASSOC_INFO=$(medperf association ls | head -n 4 | tail -n 1 | tr -s ' ')
ASSOC_DSET_UID=$(echo $ASSOC_INFO | cut -d ' ' -f 1)
ASSOC_BMK_UID=$(echo $ASSOC_INFO | cut -d ' ' -f 2)
# Mark dataset-benchmark association as approved
medperf association approve -b $ASSOC_BMK_UID -d $ASSOC_DSET_UID
print_eval medperf association approve -b $ASSOC_BMK_UID -d $ASSOC_DSET_UID
checkFailed "Association approval failed"

echo "====================================="
echo "Running benchmark execution step"
echo "====================================="
# log back as user
medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"
# Create results
medperf run -b 1 -d $DSET_UID -m 4 -y
print_eval medperf run -b 1 -d $DSET_UID -m 4 -y
checkFailed "Benchmark execution step failed"

# Test offline compatibility test
wget -P $MODEL_LOCAL/workspace/additional_files "https://storage.googleapis.com/medperf-storage/chestxray_tutorial/cnn_weights.tar.gz"
tar -xzvf $MODEL_LOCAL/workspace/additional_files/cnn_weights.tar.gz -C $MODEL_LOCAL/workspace/additional_files
medperf test run --offline --no-cache \
print_eval wget -P $MODEL_LOCAL/workspace/additional_files "https://storage.googleapis.com/medperf-storage/chestxray_tutorial/cnn_weights.tar.gz"
print_eval tar -xzvf $MODEL_LOCAL/workspace/additional_files/cnn_weights.tar.gz -C $MODEL_LOCAL/workspace/additional_files
print_eval medperf test run --offline --no-cache \
--demo_dataset_url https://storage.googleapis.com/medperf-storage/chestxray_tutorial/demo_data.tar.gz \
--demo_dataset_hash "71faabd59139bee698010a0ae3a69e16d97bc4f2dde799d9e187b94ff9157c00" \
-p $PREP_LOCAL \
-m $MODEL_LOCAL \
-e $METRIC_LOCAL

checkFailed "offline compatibility test execution step failed"
rm $MODEL_LOCAL/workspace/additional_files/cnn_weights.tar.gz
rm $MODEL_LOCAL/workspace/additional_files/cnn_weights.pth
print_eval rm $MODEL_LOCAL/workspace/additional_files/cnn_weights.tar.gz
print_eval rm $MODEL_LOCAL/workspace/additional_files/cnn_weights.pth

echo "====================================="
echo "Logout users"
echo "====================================="
medperf profile activate testbenchmark
print_eval medperf profile activate testbenchmark
checkFailed "testbenchmark profile activation failed"

medperf auth logout
print_eval medperf auth logout
checkFailed "logout failed"

medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"

medperf auth logout
print_eval medperf auth logout
checkFailed "logout failed"

echo "====================================="
echo "Delete test profiles"
echo "====================================="
medperf profile activate default
print_eval medperf profile activate default
checkFailed "default profile activation failed"

medperf profile delete testbenchmark
print_eval medperf profile delete testbenchmark
checkFailed "Profile deletion failed"

medperf profile delete testdata
print_eval medperf profile delete testdata
checkFailed "Profile deletion failed"

if ${CLEANUP}; then
Expand Down
72 changes: 38 additions & 34 deletions cli/cli_test_manual_prep.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
echo "=========================================="
echo "Printing MedPerf version"
echo "=========================================="
medperf --version
print_eval medperf --version
checkFailed "MedPerf version failed"
##########################################################

Expand All @@ -20,14 +20,14 @@ echo "\n"
echo "=========================================="
echo "Creating test profiles for each user"
echo "=========================================="
medperf profile activate local
print_eval medperf profile activate local
checkFailed "local profile creation failed"

medperf profile create -n testbenchmark
print_eval medperf profile create -n testbenchmark
checkFailed "testbenchmark profile creation failed"
medperf profile create -n testmodel
print_eval medperf profile create -n testmodel
checkFailed "testmodel profile creation failed"
medperf profile create -n testdata
print_eval medperf profile create -n testdata
checkFailed "testdata profile creation failed"
##########################################################

Expand All @@ -51,22 +51,22 @@ echo "\n"
echo "=========================================="
echo "Login each user"
echo "=========================================="
medperf profile activate testbenchmark
print_eval medperf profile activate testbenchmark
checkFailed "testbenchmark profile activation failed"

medperf auth login -e $BENCHMARKOWNER
print_eval medperf auth login -e $BENCHMARKOWNER
checkFailed "testbenchmark login failed"

medperf profile activate testmodel
print_eval medperf profile activate testmodel
checkFailed "testmodel profile activation failed"

medperf auth login -e $MODELOWNER
print_eval medperf auth login -e $MODELOWNER
checkFailed "testmodel login failed"

medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"

medperf auth login -e $DATAOWNER
print_eval medperf auth login -e $DATAOWNER
checkFailed "testdata login failed"
##########################################################

Expand All @@ -76,7 +76,7 @@ echo "\n"
echo "====================================="
echo "Activate modelowner profile"
echo "====================================="
medperf profile activate testmodel
print_eval medperf profile activate testmodel
checkFailed "testmodel profile activation failed"
##########################################################

Expand All @@ -89,9 +89,10 @@ echo "====================================="

PREP_MLCUBE="https://raw.githubusercontent.com/aristizabal95/medperf-2/4aea7de62fd71b377fd3a0b58352d104fd8f9c08/examples/DataPrepManualSteps/data_prep/mlcube/mlcube.yaml"
PREP_PARAMS="https://raw.githubusercontent.com/aristizabal95/medperf-2/4aea7de62fd71b377fd3a0b58352d104fd8f9c08/examples/DataPrepManualSteps/data_prep/mlcube/workspace/parameters.yaml"
medperf mlcube submit --name manprep -m $PREP_MLCUBE -p $PREP_PARAMS
print_eval medperf mlcube submit --name manprep -m $PREP_MLCUBE -p $PREP_PARAMS
checkFailed "Prep submission failed"
PREP_UID=$(medperf mlcube ls | tail -n 1 | tr -s ' ' | cut -d ' ' -f 2)
echo "PREP_UID=$PREP_UID"

##########################################################

Expand All @@ -101,7 +102,7 @@ echo "\n"
echo "====================================="
echo "Activate dataowner profile"
echo "====================================="
medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"
##########################################################

Expand All @@ -111,9 +112,11 @@ echo "\n"
echo "====================================="
echo "Running data creation step"
echo "====================================="
medperf dataset submit -p $PREP_UID -d $DATA_PATH -l $LABELS_PATH --name="manual_a" --description="mock manual a" --location="mock location a" -y
print_eval "medperf dataset submit -p $PREP_UID -d $DATA_PATH -l $LABELS_PATH --name='manual_a' --description='mock manual a' --location='mock location a' -y"
checkFailed "Data submission step failed"
DSET_A_UID=$(medperf dataset ls | grep manual_a | tr -s ' ' | cut -d ' ' -f 2)
echo "DSET_A_UID=$DSET_A_UID"

##########################################################

echo "\n"
Expand All @@ -122,7 +125,7 @@ echo "\n"
echo "====================================="
echo "Running data preparation step (it will fail, needs manual steps)"
echo "====================================="
medperf dataset prepare -d $DSET_A_UID -y
print_eval medperf dataset prepare -d $DSET_A_UID -y
checkSucceeded "Data preparation step should fail"
##########################################################

Expand All @@ -132,7 +135,7 @@ echo "\n"
echo "====================================="
echo "Running data preparation step (another way)"
echo "====================================="
echo "y" | medperf dataset prepare -d $DSET_A_UID
print_eval echo "y" | medperf dataset prepare -d $DSET_A_UID
checkSucceeded "Data preparation step should fail"
##########################################################

Expand All @@ -142,7 +145,7 @@ echo "\n"
echo "====================================="
echo "Running data preparation step (another way)"
echo "====================================="
echo "n" | medperf dataset prepare -d $DSET_A_UID
print_eval echo "n" | medperf dataset prepare -d $DSET_A_UID
checkSucceeded "Data preparation step should fail"
##########################################################

Expand All @@ -152,7 +155,7 @@ echo "\n"
echo "====================================="
echo "Do manual step of the preparation"
echo "====================================="
sed -i 's/0$/1/' $MEDPERF_STORAGE/data/$SERVER_STORAGE_ID/$DSET_A_UID/data/data.csv
print_eval "sed -i 's/0$/1/' $MEDPERF_STORAGE/data/$SERVER_STORAGE_ID/$DSET_A_UID/data/data.csv"
checkFailed "manual step failed"
##########################################################

Expand All @@ -162,7 +165,7 @@ echo "\n"
echo "====================================="
echo "Running data preparation step again (this will succeed)"
echo "====================================="
medperf dataset prepare -d $DSET_A_UID -y
print_eval medperf dataset prepare -d $DSET_A_UID -y
checkFailed "Data preparation step failed"
##########################################################

Expand All @@ -172,7 +175,7 @@ echo "\n"
echo "====================================="
echo "Running data set operational step"
echo "====================================="
medperf dataset set_operational -d $DSET_A_UID -y
print_eval medperf dataset set_operational -d $DSET_A_UID -y
checkFailed "Data activattion step failed"
##########################################################

Expand All @@ -182,9 +185,10 @@ echo "\n"
echo "====================================="
echo "Running data creation step"
echo "====================================="
medperf dataset submit -p $PREP_UID -d $PREPARED_DATA_PATH -l $PREPARED_LABELS_PATH --name="already_a" --description="mock already a" --location="mock location a" -y --submit-as-prepared
print_eval "medperf dataset submit -p $PREP_UID -d $PREPARED_DATA_PATH -l $PREPARED_LABELS_PATH --name='already_a' --description='mock already a' --location='mock location a' -y --submit-as-prepared"
checkFailed "Data submission step failed"
DSET_A_UID=$(medperf dataset ls | grep already_a | tr -s ' ' | cut -d ' ' -f 2)
echo "DSET_A_UID=$DSET_A_UID"
##########################################################

echo "\n"
Expand All @@ -193,7 +197,7 @@ echo "\n"
echo "====================================="
echo "Running data preparation step"
echo "====================================="
medperf dataset prepare -d $DSET_A_UID
print_eval medperf dataset prepare -d $DSET_A_UID
checkFailed "Data preparation step failed"
##########################################################

Expand All @@ -203,7 +207,7 @@ echo "\n"
echo "====================================="
echo "Running data set operational step"
echo "====================================="
medperf dataset set_operational -d $DSET_A_UID -y
print_eval medperf dataset set_operational -d $DSET_A_UID -y
checkFailed "Data activattion step failed"
##########################################################

Expand All @@ -213,22 +217,22 @@ echo "\n"
echo "====================================="
echo "Logout users"
echo "====================================="
medperf profile activate testbenchmark
print_eval medperf profile activate testbenchmark
checkFailed "testbenchmark profile activation failed"

medperf auth logout
print_eval medperf auth logout
checkFailed "logout failed"

medperf profile activate testmodel
print_eval medperf profile activate testmodel
checkFailed "testmodel profile activation failed"

medperf auth logout
print_eval medperf auth logout
checkFailed "logout failed"

medperf profile activate testdata
print_eval medperf profile activate testdata
checkFailed "testdata profile activation failed"

medperf auth logout
print_eval medperf auth logout
checkFailed "logout failed"
##########################################################

Expand All @@ -238,16 +242,16 @@ echo "\n"
echo "====================================="
echo "Delete test profiles"
echo "====================================="
medperf profile activate default
print_eval medperf profile activate default
checkFailed "default profile activation failed"

medperf profile delete testbenchmark
print_eval medperf profile delete testbenchmark
checkFailed "Profile deletion failed"

medperf profile delete testmodel
print_eval medperf profile delete testmodel
checkFailed "Profile deletion failed"

medperf profile delete testdata
print_eval medperf profile delete testdata
checkFailed "Profile deletion failed"
##########################################################

Expand Down
Loading
Loading