moving machines destroy/unlock to after_script (demisto#29789)

* moving machines destroy/unlock to after_script

.gitlab/ci/.gitlab-ci.bucket-upload.yml

@@ -268,11 +268,8 @@ install-packs-in-server-master:
     - job-done
     - exit "$EXIT_CODE"
   after_script:
-    - echo "Job finished, removing lock file"
-    - gcloud auth activate-service-account --key-file="$GCS_ARTIFACTS_KEY" > $ARTIFACTS_FOLDER/logs/gauth.out 2>$ARTIFACTS_FOLDER/logs/gauth.err
-    - gsutil rm "gs://xsoar-ci-artifacts/$GCS_LOCKS_PATH/machines_locks/*-lock-$CI_JOB_ID"
-    - echo "Finished removing lock file"
-
+    - source .gitlab/helper_functions.sh
+    - !reference [ .unlock-machine ]
 
 #install-packs-in-xsiam-dev:
 #  extends: .install-mpv2-packs-on-xsiam-instances

.gitlab/ci/.gitlab-ci.build-machines-cleanup.yml

@@ -21,9 +21,11 @@ build-machines-cleanup:
     - !reference [.lock-machine]
     - !reference [.uninstall-packs-and-reset-bucket-cloud]
     - !reference [.cloud-machine-information]
-    - !reference [.unlock-machine]
     - job-done
     - exit "${EXIT_CODE}"
+  after_script:
+    - source .gitlab/helper_functions.sh
+    - !reference [ .unlock-machine ]
   variables:
     CLOUD_MACHINES_TYPE: "nightly"
     CLOUD_MACHINES_COUNT: "all"

.gitlab/ci/.gitlab-ci.global.yml

@@ -68,12 +68,6 @@
   - python3 ./Tests/scripts/add_secrets_file_to_build.py -sa "$GSM_SERVICE_ACCOUNT" -sf "$SECRET_CONF_PATH" -u "$DEMISTO_USERNAME" -p "$DEMISTO_PASSWORD" --gsm_project_id_dev "$GSM_PROJECT_ID_DEV" --gsm_project_id_prod "$GSM_PROJECT_ID" >> $ARTIFACTS_FOLDER/logs/handle_secrets.log
   - section_end "Secrets Fetch"
 
-.ssh-config-setup:
-  - section_start "SSH config setup" --collapsed
-  - cp $GCP_SSH_CONFIGURATION ~/.ssh/config
-  - chmod 700 ~/.ssh/config
-  - section_end "SSH config setup"
-
 .check_build_files_are_up_to_date: &check_build_files_are_up_to_date
   - section_start "Check Build Files Are Up To Date"
   - |
@@ -90,6 +84,12 @@
     fi
   - section_end "Check Build Files Are Up To Date"
 
+.cleanup_logs_directory: &cleanup_logs_directory
+  - section_start "Creating new clean logs directory" --collapsed
+  - rm -rf $ARTIFACTS_FOLDER/logs
+  - mkdir -p $ARTIFACTS_FOLDER/logs
+  - section_end "Creating new clean logs directory"
+
 .stop_contrib_external_build: &stop_contrib_external_build
   - |
     if [[ $CI_COMMIT_BRANCH =~ ^contrib/* && $CI_PIPELINE_SOURCE = "push" && $(curl --header "Accept: application/vnd.github.v3.raw" --header "Authorization: token $GITHUB_TOKEN" "https://api.github.com/repos/demisto/content/pulls?state=open&base=master&head=demisto:${CI_COMMIT_BRANCH}" | jq) = '[]' ]]; then
@@ -99,17 +99,20 @@
     fi
 
 .clone_and_export_variables: &clone_and_export_variables
-  - source .gitlab/helper_functions.sh
   - section_start "Git - Job Start Actions" --collapsed
   - git checkout -B $CI_COMMIT_BRANCH $CI_COMMIT_SHA
   - git config diff.renameLimit 6000
   - section_end "Git - Job Start Actions"
+  - section_start "Create artifacts directory" --collapsed
   - mkdir -p -m 777 $ARTIFACTS_FOLDER/
+  - section_end "Create artifacts directory"
+  - section_start "Source BASH Environment" --collapsed
   - |
     if [[ -f "$BASH_ENV" ]]; then
       source "$BASH_ENV"
     fi
   - source .circleci/content_release_vars.sh
+  - section_end "Source BASH Environment"
   - section_start "Granting execute permissions on files" --collapsed
   - chmod +x ./Tests/scripts/*
   - chmod +x ./Tests/Marketplace/*
@@ -145,6 +148,12 @@
       python3 -m pip list | tee -a $ARTIFACTS_FOLDER/installed_python_libraries.txt
   - section_end "Installing Virtualenv"
 
+.ssh-config-setup:
+  - section_start "SSH config setup" --collapsed
+  - cp $GCP_SSH_CONFIGURATION ~/.ssh/config
+  - chmod 700 ~/.ssh/config
+  - section_end "SSH config setup"
+
 .install_ssh_keys: &install_ssh_keys
   - section_start "Installing SSH keys" --collapsed
   - eval $(ssh-agent -s)
@@ -157,11 +166,11 @@
 .install_node_modules: &install_node_modules
   - section_start "Installing node modules" --collapsed
   - source $NVM_DIR/nvm.sh
-  - nvm use default
-  - echo "Installing Node Modules" | tee --append $ARTIFACTS_FOLDER/logs/installations.log
-  - npm ci --cache .npm --prefer-offline | tee --append $ARTIFACTS_FOLDER/logs/installations.log
-  - npm list --json
-  - npm link jsdoc-to-markdown@5.0.3 | tee --append $ARTIFACTS_FOLDER/logs/installations.log  # disable-secrets-detection
+  - nvm use default | tee --append $ARTIFACTS_FOLDER/logs/installations_node.log
+  - echo "Installing Node Modules" | tee --append $ARTIFACTS_FOLDER/logs/installations_node.log
+  - npm ci --cache .npm --prefer-offline | tee --append $ARTIFACTS_FOLDER/logs/installations_node.log
+  - npm list --json | tee --append $ARTIFACTS_FOLDER/logs/installations_node.log
+  - npm link jsdoc-to-markdown@5.0.3 | tee --append $ARTIFACTS_FOLDER/logs/installations_node.log  # disable-secrets-detection
   - section_end "Installing node modules"
 
 .get_last_upload_commit: &get_last_upload_commit
@@ -199,10 +208,7 @@
     - *stop_contrib_external_build
     - *clone_and_export_variables
     - *check_build_files_are_up_to_date
-    - section_start "Creating new clean logs folder" --collapsed
-    - rm -rf $ARTIFACTS_FOLDER/logs
-    - mkdir -p $ARTIFACTS_FOLDER/logs
-    - section_end "Creating new clean logs folder"
+    - *cleanup_logs_directory
     - *install_node_modules
     - *install_venv
     - *get_contribution_pack
@@ -211,6 +217,13 @@
     - *neo4j-setup
     - *build_parameters
 
+.default-after-script:
+  - source .gitlab/helper_functions.sh
+  - *setup-network-certs
+  - *setup-artifactory
+  - *install_node_modules
+  - *install_venv
+
 .default-job-settings:
   interruptible: true
   extends:
@@ -231,17 +244,22 @@
   - echo "Auth done successfully"
   - ./Tests/scripts/wait_in_line_for_cloud_env.sh "$CLOUD_MACHINES_TYPE"
   - source CloudEnvVariables
-  - echo "CLOUD Chosen machine ids are:${CLOUD_CHOSEN_MACHINE_IDS}"
+  - echo "CLOUD Chosen machine ids are:${CLOUD_CHOSEN_MACHINE_IDS}" | tee $ARTIFACTS_FOLDER/logs/lock_file.txt
   - section_end "Lock Machine"
 
 .unlock-machine:
   - section_start "Unlock Machine" --collapsed
+  - source CloudEnvVariables
   - |
-    if ! [ -z "${CLOUD_CHOSEN_MACHINE_IDS}" ]; then
-      echo "Job finished, removing lock file for machine ids:${CLOUD_CHOSEN_MACHINE_IDS}"
-      gcloud auth activate-service-account --key-file="$GCS_ARTIFACTS_KEY" > $ARTIFACTS_FOLDER/logs/gauth.out 2>$ARTIFACTS_FOLDER/logs/gauth.err
-      gsutil rm "gs://xsoar-ci-artifacts/$GCS_LOCKS_PATH/machines_locks/*-lock-$CI_JOB_ID"
-      echo "Finished removing lock file"
+    if [[ -n "${CLOUD_CHOSEN_MACHINE_IDS}" ]]; then
+      if [[ -e "$ARTIFACTS_FOLDER/logs/lock_file.txt" ]]; then
+        echo "Job finished, removing lock file for machine ids:${CLOUD_CHOSEN_MACHINE_IDS}"
+        gcloud auth activate-service-account --key-file="$GCS_ARTIFACTS_KEY" > $ARTIFACTS_FOLDER/logs/gauth.out 2>$ARTIFACTS_FOLDER/logs/gauth.err
+        gsutil rm "gs://xsoar-ci-artifacts/$GCS_LOCKS_PATH/machines_locks/*-lock-$CI_JOB_ID"
+        echo "Finished removing lock file"
+      else
+        echo "No lock file found, skipping unlocking"
+      fi
     else
       echo "No machine ids were chosen, skipping unlocking"
     fi

.gitlab/ci/.gitlab-ci.on-push.yml

@@ -295,15 +295,18 @@ xpanse-prepare-testing-bucket:
         cp ./Tests/test_playbooks_report.json $ARTIFACTS_FOLDER/test_playbooks_report.json
       fi
     - section_end "Run Tests"
+    - job-done
+    - exit $EXIT_CODE
+  after_script:
+    - !reference [.default-after-script]
+    - !reference [.install_ssh_keys]
+    - !reference [.ssh-config-setup]
     - section_start "Destroy Instances"
-    - |
-      if [ -z "${TIME_TO_LIVE}" -a "$CI_PIPELINE_SOURCE" = "contrib" ]; then
-        TIME_TO_LIVE=300
+    - python3 ./Tests/scripts/destroy_instances.py --artifacts-dir $ARTIFACTS_FOLDER --env-file $ARTIFACTS_FOLDER/env_results.json --instance-role "$INSTANCE_ROLE"
+    - if [ $? -ne 0 ]; then
+        echo "Failed to destroy instances, exit code"$?";
       fi
-      python3 ./Tests/scripts/destroy_instances.py --artifacts-dir $ARTIFACTS_FOLDER --env-file $ARTIFACTS_FOLDER/env_results.json --instance-role "$INSTANCE_ROLE" || EXIT_CODE=$?
     - section_end "Destroy Instances"
-    - job-done
-    - exit $EXIT_CODE
 
 tests_xsoar_server:
   extends:
@@ -438,9 +441,11 @@ slack-notify-nightly-build:
     - section_end "Test Modeling Rules"
 
     - !reference [.cloud-machine-information]
-    - !reference [.unlock-machine]
     - job-done
     - exit $EXIT_CODE
+  after_script:
+    - source .gitlab/helper_functions.sh
+    - !reference [.unlock-machine]
 
 #xsiam_server_dev:
 #  extends: