Merge remote-tracking branch 'upstream/master' into expand-integratio…

…n-commands

.gitlab/ci/.gitlab-ci.bucket-upload.yml

@@ -132,8 +132,8 @@ xpanse-prepare-testing-bucket-upload-flow:
   artifacts:
     expire_in: 48 hrs
     paths:
-      - /builds/xsoar/content/artifacts/*
-      - /builds/xsoar/content/pipeline_jobs_folder/*
+      - ${CI_PROJECT_DIR}/artifacts/*
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
     when: always
   variables:
     ARTIFACTS_FOLDER: "${ARTIFACTS_FOLDER_XSOAR}"
@@ -207,8 +207,8 @@ install-packs-in-server-master:
   artifacts:
     expire_in: 48 hrs
     paths:
-      - /builds/xsoar/content/artifacts/*
-      - /builds/xsoar/content/pipeline_jobs_folder/*
+      - ${CI_PROJECT_DIR}/artifacts/*
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
     when: always
   variables:
     ARTIFACTS_FOLDER: "${ARTIFACTS_FOLDER_MPV2}"
@@ -298,8 +298,8 @@ upload-packs-to-marketplace:
   artifacts:
     expire_in: 48 hrs
     paths:
-      - /builds/xsoar/content/artifacts/*
-      - /builds/xsoar/content/pipeline_jobs_folder/*
+      - ${CI_PROJECT_DIR}/artifacts/*
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
     when: always
   variables:
     MARKETPLACE_VERSION: "xsoar"
@@ -380,8 +380,8 @@ upload-packs-to-marketplace-v2:
   artifacts:
     expire_in: 48 hrs
     paths:
-      - /builds/xsoar/content/artifacts/*
-      - /builds/xsoar/content/pipeline_jobs_folder/*
+      - ${CI_PROJECT_DIR}/artifacts/*
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
     when: always
   variables:
     MARKETPLACE_VERSION: "marketplacev2"
@@ -450,8 +450,8 @@ upload-packs-to-xpanse-marketplace:
   artifacts:
     expire_in: 48 hrs
     paths:
-      - /builds/xsoar/content/artifacts/*
-      - /builds/xsoar/content/pipeline_jobs_folder/*
+      - ${CI_PROJECT_DIR}/artifacts/*
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
     when: always
   variables:
     MARKETPLACE_VERSION: "xpanse"
@@ -521,8 +521,8 @@ upload-packs-to-xsoar-saas-marketplace:
   artifacts:
     expire_in: 48 hrs
     paths:
-      - /builds/xsoar/content/artifacts/*
-      - /builds/xsoar/content/pipeline_jobs_folder/*
+      - ${CI_PROJECT_DIR}/artifacts/*
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
     when: always
   variables:
     MARKETPLACE_VERSION: "xsoar_saas"

.gitlab/ci/.gitlab-ci.global.yml

@@ -281,7 +281,7 @@
   - |
     if [[ "${CI_COMMIT_BRANCH}" = "master" ]]; then
       echo "Skipping, Should not run on master branch."
-    elif [ 'true' = $(./Tests/scripts/check_if_branch_exist.sh -u "gitlab-ci-token" -t "${CI_JOB_TOKEN}" -h "code.pan.run" --repo "xsoar/content-test-conf" -b "${CI_COMMIT_BRANCH}") ]; then
+    elif [ 'true' = $(./Tests/scripts/check_if_branch_exist.sh -u "gitlab-ci-token" -t "${CI_JOB_TOKEN}" -h "${CI_SERVER_HOST}" --repo "${CI_PROJECT_NAMESPACE}/content-test-conf" -b "${CI_COMMIT_BRANCH}") ]; then
       RED='\033[0;31m'
       NC='\033[0m'
       echo -e "${RED}ERROR: Found a branch with the same name:${CI_COMMIT_BRANCH} in contest-test-conf repository.\n Merge it in order to merge the current branch into content repo.${NC}"
@@ -301,15 +301,15 @@
     reports:
       coverage_report:
         coverage_format: cobertura
-        path: /builds/xsoar/content/artifacts/coverage_report/coverage.xml
+        path: ${CI_PROJECT_DIR}/artifacts/coverage_report/coverage.xml
     expire_in: 30 days
     paths:
-      - /builds/xsoar/content/unit-tests
-      - /builds/xsoar/content/artifacts/*
-      - /builds/xsoar/content/pipeline_jobs_folder/*
+      - ${CI_PROJECT_DIR}/unit-tests
+      - ${CI_PROJECT_DIR}/artifacts/*
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
     when: always
   services:
-    - name: docker.art.code.pan.run/build-tools--image-dind:20.10.12-dind
+    - name: ${DOCKER_IO}/library/docker:20.10.12-dind
       alias: docker
   variables:
     DOCKER_HOST: tcp://docker:2375
@@ -398,8 +398,8 @@
   artifacts:
     expire_in: 30 days
     paths:
-      - /builds/xsoar/content/artifacts/*
-      - /builds/xsoar/content/pipeline_jobs_folder/*
+      - ${CI_PROJECT_DIR}/artifacts/*
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
     when: always
   script:
     - section_start "Look For Secrets"

.gitlab/ci/.gitlab-ci.on-push.yml

@@ -510,8 +510,8 @@ xsiam_server_ga:
         - ${ARTIFACTS_FOLDER}/modeling_rules_results.xml
     paths:
       - ${ARTIFACTS_FOLDER}/modeling_rules_results.xml
-      - /builds/xsoar/content/artifacts/*  # restoring the default artifacts path from the job default settings
-      - /builds/xsoar/content/pipeline_jobs_folder/*  # restoring the default artifacts path from the job default settings
+      - ${CI_PROJECT_DIR}/artifacts/*  # restoring the default artifacts path from the job default settings
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*  # restoring the default artifacts path from the job default settings
 
 xsoar_ng_server_ga:
   extends:
@@ -577,7 +577,7 @@ test-upload-flow:
     -   echo "Sleeping for 10 seconds before retrying"
     -   sleep 10
     - done
-    - echo "Successful triggered test upload - https://code.pan.run/xsoar/content/-/pipelines/$pipeline_id"  # disable-secrets-detection
+    - echo "Successful triggered test upload - ${CI_SERVER_URL}/${CI_PROJECT_NAMESPACE}/content/-/pipelines/$pipeline_id"  # disable-secrets-detection
     - section_end "Trigger Test Upload Flow On Testing Branch"
 
     - section_start "Wait For Upload To Finish"

.gitlab/ci/.gitlab-ci.sdk-nightly.yml

@@ -161,7 +161,7 @@ demisto-sdk-nightly:run-commands-against-instance:
     - .default-job-settings
     - .sdk-nightly-schedule-rule-always
   services: # needed for running lint
-    - name: docker.art.code.pan.run/build-tools--image-dind:20.10.12-dind
+    - name: ${DOCKER_IO}/library/docker:20.10.12-dind
       alias: docker
   variables:
     DOCKER_HOST: tcp://docker:2375
@@ -219,7 +219,7 @@ demisto-sdk-nightly:run-end-to-end-tests:
     - .default-job-settings
     - .sdk-nightly-schedule-rule-always
   services: # needed for running e2e tests for docker
-    - name: docker.art.code.pan.run/build-tools--image-dind:20.10.12-dind
+    - name: ${DOCKER_IO}/library/docker:20.10.12-dind
       alias: docker
   variables:
     DOCKER_HOST: tcp://docker:2375

.gitlab/ci/.gitlab-ci.security-scans.yml

@@ -18,7 +18,7 @@ auto secure cicd:
     include:
       - file: "/.gitlab/ci/security-scans.yml"
         ref: master
-        project: "xsoar/infra"
+        project: "${CI_PROJECT_NAMESPACE}/infra"
     strategy: depend
   extends:
     .auto-secure-cicd-rule

.gitlab/ci/.gitlab-ci.slack-notify.yml

@@ -1,11 +1,11 @@
 
 default:
-  image: docker-io.art.code.pan.run/devdemisto/gitlab-content-ci:1.0.0.64455
+  image: ${DOCKER_IO}/devdemisto/gitlab-content-ci:1.0.0.64455
   artifacts:
     expire_in: 30 days
     paths:
-      - /builds/xsoar/content/artifacts/*
-      - /builds/xsoar/content/pipeline_jobs_folder/*
+      - ${CI_PROJECT_DIR}/artifacts/*
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
     when: always
 
 stages:

.gitlab/ci/.gitlab-ci.yml

@@ -1,11 +1,11 @@
 
 default:
-  image: docker-io.art.code.pan.run/devdemisto/gitlab-content-ci:1.0.0.64455
+  image: ${DOCKER_IO}/devdemisto/gitlab-content-ci:1.0.0.64455
   artifacts:
     expire_in: 30 days
     paths:
-      - /builds/xsoar/content/artifacts/*
-      - /builds/xsoar/content/pipeline_jobs_folder/*
+      - ${CI_PROJECT_DIR}/artifacts/*
+      - ${CI_PROJECT_DIR}/pipeline_jobs_folder/*
     when: always
 
 stages:
@@ -33,20 +33,20 @@ variables:
   STORAGE_BASE_PATH: ""
   SLACK_CHANNEL: "dmst-build-test"
   DEMISTO_README_VALIDATION: "true"
-  ARTIFACTS_FOLDER: "/builds/xsoar/content/artifacts"
-  PIPELINE_JOBS_FOLDER: "/builds/xsoar/content/pipeline_jobs_folder"
-  ARTIFACTS_FOLDER_XSOAR: "/builds/xsoar/content/artifacts/xsoar"
-  ARTIFACTS_FOLDER_MPV2: "/builds/xsoar/content/artifacts/marketplacev2"
-  ARTIFACTS_FOLDER_XPANSE: "/builds/xsoar/content/artifacts/xpanse"
-  ARTIFACTS_FOLDER_XSOAR_SAAS: "/builds/xsoar/content/artifacts/xsoar_saas"
-  BASH_ENV: "/builds/xsoar/content/artifacts/bash_env"
-  PYTHONPATH: "/builds/xsoar/content"
+  ARTIFACTS_FOLDER: "${CI_PROJECT_DIR}/artifacts"
+  PIPELINE_JOBS_FOLDER: "${CI_PROJECT_DIR}/pipeline_jobs_folder"
+  ARTIFACTS_FOLDER_XSOAR: "${CI_PROJECT_DIR}/artifacts/xsoar"
+  ARTIFACTS_FOLDER_MPV2: "${CI_PROJECT_DIR}/artifacts/marketplacev2"
+  ARTIFACTS_FOLDER_XPANSE: "${CI_PROJECT_DIR}/artifacts/xpanse"
+  ARTIFACTS_FOLDER_XSOAR_SAAS: "${CI_PROJECT_DIR}/artifacts/xsoar_saas"
+  BASH_ENV: "${CI_PROJECT_DIR}/artifacts/bash_env"
+  PYTHONPATH: "${CI_PROJECT_DIR}"
   PIP_CACHE_DIR: "$CI_PROJECT_DIR/.cache/pip"
   FEATURE_BRANCH_NAME: "v4.5.0"
   IFRA_ENV_TYPE: "Content-Env"
   GIT_DEPTH: 0
   GET_SOURCES_ATTEMPTS: 3  # see https://docs.gitlab.com/ee/ci/runners/configure_runners.html#job-stages-attempts
-  ENV_RESULTS_PATH: "/builds/xsoar/content/artifacts/env_results.json"
+  ENV_RESULTS_PATH: "${CI_PROJECT_DIR}/artifacts/env_results.json"
   GCS_PRODUCTION_BUCKET: "marketplace-dist"
   GCS_PRODUCTION_V2_BUCKET: "marketplace-v2-dist"
   GCS_PRODUCTION_XPANSE_BUCKET: "xpanse-dist"
@@ -64,6 +64,7 @@ variables:
   FF_USE_FASTZIP: "true"
   POETRY_VERSION: "1.6.1"
   INSTALL_POETRY: "true"
+  DOCKER_IO: docker-io.art.code.pan.run
 
 include:
   - local: .gitlab/ci/.gitlab-ci.global.yml

Tests/Marketplace/search_and_install_packs.py

@@ -36,7 +36,7 @@
 WLM_TASK_FAILED_ERROR_CODE = 101704
 
 GITLAB_SESSION = Session()
-CONTENT_PROJECT_ID = "2596"
+CONTENT_PROJECT_ID = os.getenv('CI_PROJECT_ID', '2596')  # the default is the id of the content repo in code.pan.run
 PACKS_DIR = "Packs"
 PACK_METADATA_FILE = Pack.USER_METADATA
 GITLAB_PACK_METADATA_URL = f'{{gitlab_url}}/api/v4/projects/{CONTENT_PROJECT_ID}/repository/files/{PACKS_DIR}%2F{{pack_id}}%2F{PACK_METADATA_FILE}'  # noqa: E501

Tests/scripts/download_conf_repos.sh

@@ -102,7 +102,9 @@ echo ${DEMISTO_LIC_PATH} > demisto_lic_path
 DEMISTO_PACK_SIGNATURE_UTIL_PATH="./signDirectory"
 echo ${DEMISTO_PACK_SIGNATURE_UTIL_PATH} > demisto_pack_sig_util_path
 
-clone_repository_with_fallback_branch "code.pan.run" "gitlab-ci-token" "${CI_JOB_TOKEN}" "xsoar/content-test-conf" "${CI_COMMIT_BRANCH}" 3 10 "master"
+CI_SERVER_HOST=${CI_SERVER_HOST:-code.pan.run}
+
+clone_repository_with_fallback_branch "${CI_SERVER_HOST}" "gitlab-ci-token" "${CI_JOB_TOKEN}" "${CI_PROJECT_NAMESPACE}/content-test-conf" "${CI_COMMIT_BRANCH}" 3 10 "master"
 
 cp ./content-test-conf/secrets_build_scripts/google_secret_manager_handler.py ./Tests/scripts
 cp ./content-test-conf/secrets_build_scripts/add_secrets_file_to_build.py ./Tests/scripts
@@ -119,7 +121,7 @@ if [[ "${NIGHTLY}" == "true" || "${EXTRACT_PRIVATE_TESTDATA}" == "true" ]]; then
 fi
 rm -rf ./content-test-conf
 
-clone_repository_with_fallback_branch "code.pan.run" "gitlab-ci-token" "${CI_JOB_TOKEN}" "xsoar/infra" "${CI_COMMIT_BRANCH}" 3 10 "master"
+clone_repository_with_fallback_branch "${CI_SERVER_HOST}" "gitlab-ci-token" "${CI_JOB_TOKEN}" "${CI_PROJECT_NAMESPACE}/infra" "${CI_COMMIT_BRANCH}" 3 10 "master"
 
 cp -r ./infra/xsiam_servers.json $XSIAM_SERVERS_PATH
 cp -r ./infra/xsoar_ng_servers.json $XSOAR_NG_SERVERS_PATH

Tests/scripts/lock_cloud_machines.py

@@ -7,12 +7,14 @@
 import requests
 from google.cloud import storage  # noqa
 import argparse
+from Utils.github_workflow_scripts.utils import get_env_var
 
+GITLAB_SERVER_URL = get_env_var('CI_SERVER_URL', 'https://code.pan.run')  # disable-secrets-detection
 LOCKS_BUCKET = 'xsoar-ci-artifacts'
 QUEUE_REPO = 'queue'
 MACHINES_LOCKS_REPO = 'machines_locks'
-JOB_STATUS_URL = 'https://code.pan.run/api/v4/projects/{}/jobs/{}'  # disable-secrets-detection
-CONTENT_GITLAB_PROJECT_ID = '2596'
+JOB_STATUS_URL = '{}/api/v4/projects/{}/jobs/{}'  # disable-secrets-detection
+CONTENT_GITLAB_PROJECT_ID = get_env_var('CI_PROJECT_ID', '2596')  # the default is the id of the content repo in code.pan.run
 
 
 def options_handler() -> argparse.Namespace:
@@ -99,7 +101,7 @@ def check_job_status(token: str, job_id: str, num_of_retries: int = 5, interval:
     Returns: the status of the job.
 
     """
-    user_endpoint = JOB_STATUS_URL.format(CONTENT_GITLAB_PROJECT_ID, job_id)
+    user_endpoint = JOB_STATUS_URL.format(GITLAB_SERVER_URL, CONTENT_GITLAB_PROJECT_ID, job_id)
     headers = {'PRIVATE-TOKEN': token}
 
     for attempt_num in range(1, num_of_retries + 1):

Utils/github_workflow_scripts/trigger_contribution_build.py

@@ -1,12 +1,13 @@
 #!/usr/bin/env python3
-
+import os
 import argparse
 import sys
 
 import requests
 import urllib3
 
 urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+GITLAB_PROJECT_ID = os.getenv('CI_PROJECT_ID', '2596')  # the default is the id of the content project in code.pan.run
 
 
 def arguments_handler():
@@ -39,7 +40,7 @@ def trigger_generic_webhook(options):
     body = {
         "name": "GenericWebhook_trigger_contribution_build",
         "raw_json": {"BaseBranch": base_branch, "PullRequestNumber": pr_number, "ContribBranch": contrib_branch,
-                     "ProjectID": "2596"},
+                     "ProjectID": GITLAB_PROJECT_ID},
     }
     # post to Content Gold
     res = requests.post(contribution_build_instance_url, json=body, auth=(username, password))

Utils/gitlab/stop_running_pipelines.sh

@@ -4,7 +4,9 @@
 set -e
 
 # set local vars
-CONTENT_PIPELINES_API_URL=https://code.pan.run/api/v4/projects/2596/pipelines # disable-secrets-detection
+CI_SERVER_URL=${CI_SERVER_URL:-https://code.pan.run} # disable-secrets-detection
+CI_PROJECT_ID=${CI_PROJECT_ID:-2596}
+CONTENT_PIPELINES_API_URL=${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/pipelines
 
 if [ -z "$1" ]; then
   echo "No commit branch. Aborting."

Utils/gitlab_triggers/trigger_build_url.sh

@@ -1 +1,3 @@
-export BUILD_TRIGGER_URL="https://code.pan.run/api/v4/projects/2596/trigger/pipeline"  # disable-secrets-detection
+CI_SERVER_URL=${CI_SERVER_URL:-https://code.pan.run} # disable-secrets-detection
+CI_PROJECT_ID=${CI_PROJECT_ID:-2596}
+export BUILD_TRIGGER_URL="${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/trigger/pipeline"

Utils/test_upload_flow/create_test_branch.py

@@ -5,23 +5,25 @@
 import subprocess
 import time
 from pathlib import Path
-from typing import Union
 from git import GitCommandError, Head, Repo
 from zipfile import ZipFile
 from packaging.version import Version
 
 from Tests.scripts.utils import logging_wrapper as logging
 from Tests.scripts.utils.log_util import install_logging
+from Utils.github_workflow_scripts.utils import get_env_var
 
 versions_dict = {}
 pack_items_dict = {}
 changed_packs = set()
 
+GITLAB_SERVER_HOST = get_env_var('CI_SERVER_HOST', 'code.pan.run')  # disable-secrets-detection
+GITLAB_PROJECT_NAMESPACE = get_env_var('CI_PROJECT_NAMESPACE', 'xsoar')  # disable-secrets-detection
 
 # HELPER FUNCTIONS
 
 
-def json_write(file_path: str, data: Union[list, dict]):
+def json_write(file_path: str, data: list | dict):
     """ Writes given data to a json file
 
     Args:
@@ -366,8 +368,9 @@ def main():
         repo.git.commit(m="Added Test file", no_verify=True)
         repo.git.push('--set-upstream',
                       f'https://GITLAB_PUSH_TOKEN:{args.gitlab_mirror_token}@'  # disable-secrets-detection
-                      f'code.pan.run/xsoar/content.git', branch, push_option="ci.skip")  # disable-secrets-detection
-        logging.info("Successfuly pushing the branch to Gitlab content repo")
+                      f'{GITLAB_SERVER_HOST}/{GITLAB_PROJECT_NAMESPACE}/content.git',  # disable-secrets-detection
+                      branch, push_option="ci.skip")  # disable-secrets-detection
+        logging.info("Successfully pushed the branch to GitLab content repo")
 
     except GitCommandError as e:
         logging.error(e)

Utils/test_upload_flow/wait_for_upload.py

@@ -5,9 +5,13 @@
 import time
 import requests
 from Tests.scripts.utils.log_util import install_logging
+from Utils.github_workflow_scripts.utils import get_env_var
 
+GITLAB_SERVER_URL = get_env_var('CI_SERVER_URL', 'https://code.pan.run')  # disable-secrets-detection
+GITLAB_PROJECT_ID = get_env_var('CI_PROJECT_ID', '2596')  # the default is the id of the content project in code.pan.run
 
-GITLAB_CONTENT_PIPELINES_BASE_URL = 'http://code.pan.run/api/v4/projects/2596/pipelines/'  # disable-secrets-detection
+# disable-secrets-detection
+GITLAB_CONTENT_PIPELINES_BASE_URL = f'{GITLAB_SERVER_URL}/api/v4/projects/{GITLAB_PROJECT_ID}/pipelines/'
 TIMEOUT = 60 * 60 * 6  # 6 hours - TODO - Decrease after replacing id-set with graph
 
 
@@ -55,6 +59,7 @@ def get_job_status(job_name, pipelines_jobs_response):
     for job in pipelines_jobs_response:
         if job.get('name') == job_name:
             return job.get('status')
+    return None
 
 
 def main():