Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove chmod calls for running scripts #1489

Merged
merged 2 commits into from
Nov 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 0 additions & 3 deletions perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@ cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse"
echo "Building and installing gcsfuse"
# Get the latest commitId of yesterday in the log file. Build gcsfuse and run
commitId=$(git log --before='yesterday 23:59:59' --max-count=1 --pretty=%H)
chmod +x perfmetrics/scripts/build_and_install_gcsfuse.sh
./perfmetrics/scripts/build_and_install_gcsfuse.sh $commitId

# Mounting gcs bucket
Expand All @@ -45,12 +44,10 @@ LOG_FILE_FIO_TESTS=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs.txt
GCSFUSE_FIO_FLAGS="$GCSFUSE_FLAGS --log-file $LOG_FILE_FIO_TESTS --stackdriver-export-interval=30s"

# Executing perf tests
chmod +x run_load_test_and_fetch_metrics.sh
./run_load_test_and_fetch_metrics.sh "$GCSFUSE_FIO_FLAGS" "$UPLOAD_FLAGS"

# ls_metrics test. This test does gcsfuse mount with the passed flags first and then does the testing.
LOG_FILE_LIST_TESTS=${KOKORO_ARTIFACTS_DIR}/gcsfuse-list-logs.txt
GCSFUSE_LIST_FLAGS="$GCSFUSE_FLAGS --log-file $LOG_FILE_LIST_TESTS"
cd "./ls_metrics"
chmod +x run_ls_benchmark.sh
./run_ls_benchmark.sh "$GCSFUSE_LIST_FLAGS" "$UPLOAD_FLAGS"
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,8 @@ cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse"
echo "Building and installing gcsfuse..."
# Get the latest commitId of yesterday in the log file. Build gcsfuse and run
commitId=$(git log --before='yesterday 23:59:59' --max-count=1 --pretty=%H)
chmod +x perfmetrics/scripts/build_and_install_gcsfuse.sh
./perfmetrics/scripts/build_and_install_gcsfuse.sh $commitId

echo "Running e2e tests on installed package...."
chmod +x perfmetrics/scripts/run_e2e_tests.sh
# $1 argument is refering to value of testInstalledPackage
./perfmetrics/scripts/run_e2e_tests.sh $RUN_E2E_TESTS_ON_INSTALLED_PACKAGE
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,6 @@ END_DATE=$(echo "$config" | jq -r '.end_date')

echo "Building and installing gcsfuse"
# Get the latest commitId of yesterday in the log file. Build gcsfuse and run
chmod +x perfmetrics/scripts/build_and_install_gcsfuse.sh
./perfmetrics/scripts/build_and_install_gcsfuse.sh $BRANCH

cd "./perfmetrics/scripts/"
Expand All @@ -66,12 +65,10 @@ fi
# Executing perf tests
LOG_FILE_FIO_TESTS="${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs${EXPERIMENT_NUMBER}.txt"
GCSFUSE_FIO_FLAGS="$GCSFUSE_FLAGS --log-file $LOG_FILE_FIO_TESTS --log-format \"text\" --stackdriver-export-interval=30s"
chmod +x run_load_test_and_fetch_metrics.sh
./run_load_test_and_fetch_metrics.sh "$GCSFUSE_FIO_FLAGS" "$UPLOAD_FLAGS"

# ls_metrics test. This test does gcsfuse mount with the passed flags first and then does the testing.
LOG_FILE_LIST_TESTS="${KOKORO_ARTIFACTS_DIR}/gcsfuse-list-logs${EXPERIMENT_NUMBER}.txt"
GCSFUSE_LIST_FLAGS="$GCSFUSE_FLAGS --log-file $LOG_FILE_LIST_TESTS --log-format \"text\" --stackdriver-export-interval=30s"
cd "./ls_metrics"
chmod +x run_ls_benchmark.sh
./run_ls_benchmark.sh "$GCSFUSE_LIST_FLAGS" "$UPLOAD_FLAGS"
1 change: 0 additions & 1 deletion perfmetrics/scripts/continuous_test/ml_tests/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ source ~/.bashrc
cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts"

echo "Setting up a machine"
chmod +x ml_tests/setup.sh
source ml_tests/setup.sh

echo "Running ML model automation script"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,5 @@ TEST_SCRIPT_PATH="github/gcsfuse/perfmetrics/scripts/ml_tests/pytorch/dino/setup

cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/"

chmod +x run_and_manage_test.sh
source run_and_manage_test.sh $VM_NAME $ZONE_NAME $ARTIFACTS_BUCKET_PATH $TEST_SCRIPT_PATH

Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,5 @@ TEST_SCRIPT_PATH="github/gcsfuse/perfmetrics/scripts/ml_tests/tf/resnet/setup_ho

cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/"

chmod +x run_and_manage_test.sh
source run_and_manage_test.sh $VM_NAME $ZONE_NAME $ARTIFACTS_BUCKET_PATH $TEST_SCRIPT_PATH

1 change: 0 additions & 1 deletion perfmetrics/scripts/ml_tests/pytorch/dino/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ WORKDIR "/pytorch_dino/"
RUN git clone "https://github.com/facebookresearch/dino"

COPY perfmetrics/scripts/ml_tests/pytorch/dino/setup_container.sh ./
RUN chmod +x setup_container.sh

RUN mkdir -p "run_artifacts"
RUN mkdir -p "gcsfuse_data"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ set -e
cd "$HOME/github/gcsfuse/perfmetrics/scripts"

echo "Setting up the machine with Docker and Nvidia Driver"
chmod +x ml_tests/setup_host.sh
source ml_tests/setup_host.sh

cd "$HOME/github/gcsfuse"
Expand All @@ -20,7 +19,6 @@ sudo docker run --gpus all --name=pytorch_automation_container --privileged -d -
--shm-size=128g pytorch-gcsfuse:latest

# Setup the log_rotation.
chmod +x perfmetrics/scripts/ml_tests/setup_log_rotation.sh
source perfmetrics/scripts/ml_tests/setup_log_rotation.sh $HOME/github/gcsfuse/container_artifacts/gcsfuse.log

# Wait for the script completion as well as logs output.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,6 @@ def _run_model(directory_name, data_path, data_read_method, ml_model_path, req_f
end_time = int(time.time())

os.system(f'''cd ..
chmod +x populate_metrics.sh
./populate_metrics.sh {start_time} {end_time}
''')

Expand Down
1 change: 0 additions & 1 deletion perfmetrics/scripts/ml_tests/tf/resnet/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ WORKDIR "/tf_test/"

COPY ./perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh .
COPY ./perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/resnet_runner.py .
RUN chmod +x setup_container.sh

RUN mkdir -p "myBucket"
ENTRYPOINT ["/bin/bash", "-c", "./setup_container.sh"]
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ set -e
cd "$HOME/github/gcsfuse/perfmetrics/scripts"

echo "Setting up the machine with Docker and Nvidia Driver..."
chmod +x ml_tests/setup_host.sh
source ml_tests/setup_host.sh

cd "$HOME/github/gcsfuse/"
Expand All @@ -21,7 +20,6 @@ sudo docker run --gpus all --name tf_model_container --privileged -d \
-v $HOME/github/gcsfuse/container_artifacts/output:/home/output:rw,rshared --shm-size=24g tf-dlc-gcsfuse:latest

# Setup the log_rotation.
chmod +x perfmetrics/scripts/ml_tests/setup_log_rotation.sh
source perfmetrics/scripts/ml_tests/setup_log_rotation.sh $HOME/github/gcsfuse/container_artifacts/logs/gcsfuse.log

# Wait for the script completion as well as logs output.
Expand Down
2 changes: 0 additions & 2 deletions perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@ function execute_perf_test() {
# The VM will itself exit if the gcsfuse mount fails.
go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT
# Running FIO test
chmod +x perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh
./perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh
sudo umount gcs
}
Expand Down Expand Up @@ -115,7 +114,6 @@ then
git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER

echo "Running e2e tests...."
chmod +x perfmetrics/scripts/run_e2e_tests.sh
# $1 argument is refering to value of testInstalledPackage.
./perfmetrics/scripts/run_e2e_tests.sh $RUN_E2E_TESTS_ON_INSTALLED_PACKAGE
fi