[CI] Backport CI tests fixes [1.3.x]

mlrun · May 25, 2023 · 94e17f4 · 94e17f4
1 parent 5eb70f2
commit 94e17f4
Show file tree

Hide file tree

Showing 3 changed files with 106 additions and 82 deletions.
diff --git a/.github/workflows/system-tests-enterprise.yml b/.github/workflows/system-tests-enterprise.yml
@@ -245,26 +245,25 @@ jobs:
       timeout-minutes: 50
       run: |
         python automation/system_test/prepare.py run \
-          "${{ steps.computed_params.outputs.mlrun_version }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_IP }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_USERNAME }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_PASSWORD }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_APP_CLUSTER_SSH_PASSWORD }}" \
-          "${{ secrets.SYSTEM_TEST_GITHUB_ACCESS_TOKEN }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_PROVCTL_DOWNLOAD_PATH }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_PROVCTL_DOWNLOAD_URL_S3_ACCESS_KEY }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_PROVCTL_DOWNLOAD_URL_S3_KEY_ID }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_MLRUN_DB_PATH }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_WEBAPI_DIRECT_URL }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_FRAMESD_URL }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_USERNAME }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_ACCESS_KEY }}" \
-          "${{ steps.computed_params.outputs.iguazio_version }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_SPARK_SERVICE }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_PASSWORD }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_SLACK_WEBHOOK_URL }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_MYSQL_USER }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_MYSQL_PASSWORD }}" \
+          --mlrun-version "${{ steps.computed_params.outputs.mlrun_version }}" \
+          --data-cluster-ip "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_IP }}" \
+          --data-cluster-ssh-username "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_USERNAME }}" \
+          --data-cluster-ssh-password "${{ secrets.LATEST_SYSTEM_TEST_DATA_CLUSTER_SSH_PASSWORD }}" \
+          --app-cluster-ssh-password "${{ secrets.LATEST_SYSTEM_TEST_APP_CLUSTER_SSH_PASSWORD }}" \
+          --github-access-token "${{ secrets.SYSTEM_TEST_GITHUB_ACCESS_TOKEN }}" \
+          --provctl-download-url "${{ secrets.LATEST_SYSTEM_TEST_PROVCTL_DOWNLOAD_PATH }}" \
+          --provctl-download-s3-access-key "${{ secrets.LATEST_SYSTEM_TEST_PROVCTL_DOWNLOAD_URL_S3_ACCESS_KEY }}" \
+          --provctl-download-s3-key-id "${{ secrets.LATEST_SYSTEM_TEST_PROVCTL_DOWNLOAD_URL_S3_KEY_ID }}" \
+          --mlrun-dbpath "${{ secrets.LATEST_SYSTEM_TEST_MLRUN_DB_PATH }}" \
+          --webapi-direct-url "${{ secrets.LATEST_SYSTEM_TEST_WEBAPI_DIRECT_URL }}" \
+          --framesd-url "${{ secrets.LATEST_SYSTEM_TEST_FRAMESD_URL }}" \
+          --username "${{ secrets.LATEST_SYSTEM_TEST_USERNAME }}" \
+          --access-key "${{ secrets.LATEST_SYSTEM_TEST_ACCESS_KEY }}" \
+          --iguazio-version "${{ steps.computed_params.outputs.iguazio_version }}" \
+          --spark-service "${{ secrets.LATEST_SYSTEM_TEST_SPARK_SERVICE }}" \
+          --slack-webhook-url "${{ secrets.LATEST_SYSTEM_TEST_SLACK_WEBHOOK_URL }}" \
+          --mysql-user "${{ secrets.LATEST_SYSTEM_TEST_MYSQL_USER }}" \
+          --mysql-password "${{ secrets.LATEST_SYSTEM_TEST_MYSQL_PASSWORD }}" \
           --purge-db \
           --mlrun-commit "${{ steps.computed_params.outputs.mlrun_hash }}" \
           --override-image-registry "${{ steps.computed_params.outputs.mlrun_docker_registry }}" \
@@ -312,16 +311,16 @@ jobs:
       timeout-minutes: 5
       run: |
         python automation/system_test/prepare.py env \
-          "${{ secrets.LATEST_SYSTEM_TEST_MLRUN_DB_PATH }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_WEBAPI_DIRECT_URL }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_FRAMESD_URL }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_USERNAME }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_ACCESS_KEY }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_SPARK_SERVICE }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_PASSWORD }}" \
-          "${{ secrets.LATEST_SYSTEM_TEST_SLACK_WEBHOOK_URL }}" \
-          "${{ needs.prepare-system-tests-enterprise-ci.outputs.mlrunBranch }}"
-          "${{ secrets.SYSTEM_TEST_GITHUB_ACCESS_TOKEN }}" \
+          --mlrun-dbpath "${{ secrets.LATEST_SYSTEM_TEST_MLRUN_DB_PATH }}" \
+          --webapi-direct-url "${{ secrets.LATEST_SYSTEM_TEST_WEBAPI_DIRECT_URL }}" \
+          --framesd-url "${{ secrets.LATEST_SYSTEM_TEST_FRAMESD_URL }}" \
+          --username "${{ secrets.LATEST_SYSTEM_TEST_USERNAME }}" \
+          --access-key "${{ secrets.LATEST_SYSTEM_TEST_ACCESS_KEY }}" \
+          --spark-service "${{ secrets.LATEST_SYSTEM_TEST_SPARK_SERVICE }}" \
+          --password "${{ secrets.LATEST_SYSTEM_TEST_PASSWORD }}" \
+          --slack-webhook-url "${{ secrets.LATEST_SYSTEM_TEST_SLACK_WEBHOOK_URL }}" \
+          --branch "${{ needs.prepare-system-tests-enterprise-ci.outputs.mlrunBranch }}" \
+          --github-access-token "${{ secrets.SYSTEM_TEST_GITHUB_ACCESS_TOKEN }}"
     - name: Run System Tests
       run: |
         MLRUN_SYSTEM_TESTS_CLEAN_RESOURCES="${{ needs.prepare-system-tests-enterprise-ci.outputs.mlrunSystemTestsCleanResources }}" \

diff --git a/.github/workflows/system-tests-opensource.yml b/.github/workflows/system-tests-opensource.yml
@@ -208,7 +208,20 @@ jobs:
 
     - name: Prepare system tests env
       run: |
-        echo "MLRUN_DBPATH: http://$(minikube ip):${MLRUN_API_NODE_PORT}" > tests/system/env.yml
+        python automation/system_test/prepare.py env \
+          --mlrun-dbpath "http://$(minikube ip):${MLRUN_API_NODE_PORT}" \
+          --github-access-token "${{ secrets.SYSTEM_TEST_GITHUB_ACCESS_TOKEN }}"
+
+    # Enable tmate debugging of manually-triggered workflows if the input option was provided
+    - name: Setup tmate session
+      uses: mxschmitt/action-tmate@v3
+      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' }}
+      with:
+
+        # run in detach mode to allow the workflow to continue running while session is active
+        # this will wait up to 10 minutes AFTER the entire job is done. Once user connects to the session,
+        # it will wait until the user disconnects before finishing up the job.
+        detached: true
 
     - name: Run system tests
       timeout-minutes: 180
@@ -235,8 +248,3 @@ jobs:
         minikube kubectl -- --namespace ${NAMESPACE} get pvc
         minikube kubectl -- --namespace ${NAMESPACE} get pv
         set +x
-
-    # Enable tmate debugging of manually-triggered workflows if the input option was provided
-    - name: Setup tmate session
-      uses: mxschmitt/action-tmate@v3
-      if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' }}
diff --git a/automation/system_test/prepare.py b/automation/system_test/prepare.py
@@ -15,7 +15,9 @@
 
 import datetime
 import logging
+import os
 import pathlib
+import shutil
 import subprocess
 import sys
 import tempfile
@@ -27,8 +29,10 @@
 import paramiko
 import yaml
 
+# TODO: remove and use local logger
 import mlrun.utils
 
+project_dir = pathlib.Path(__file__).resolve().parent.parent.parent
 logger = mlrun.utils.create_logger(level="debug", name="automation")
 logging.getLogger("paramiko").setLevel(logging.DEBUG)
 
@@ -41,7 +45,9 @@ class Constants:
         igz_version_file = homedir / "igz" / "version.txt"
         mlrun_code_path = workdir / "mlrun"
         provctl_path = workdir / "provctl"
-        system_tests_env_yaml = pathlib.Path("tests") / "system" / "env.yml"
+        system_tests_env_yaml = (
+            project_dir / pathlib.Path("tests") / "system" / "env.yml"
+        )
         namespace = "default-tenant"
 
         git_url = "https://github.com/mlrun/mlrun.git"
@@ -68,7 +74,6 @@ def __init__(
         access_key: str = None,
         iguazio_version: str = None,
         spark_service: str = None,
-        password: str = None,
         slack_webhook_url: str = None,
         mysql_user: str = None,
         mysql_password: str = None,
@@ -113,8 +118,6 @@ def __init__(
             # (e.g. tests which use public repos, therefor doesn't need that access token)
             "MLRUN_SYSTEM_TESTS_GIT_TOKEN": github_access_token,
         }
-        if password:
-            self._env_config["V3IO_PASSWORD"] = password
 
     def prepare_local_env(self):
         self._prepare_env_local()
@@ -321,16 +324,20 @@ def _prepare_env_remote(self):
         )
 
     def _prepare_env_local(self):
-        contents = yaml.safe_dump(self._env_config)
         filepath = str(self.Constants.system_tests_env_yaml)
+        backup_filepath = str(self.Constants.system_tests_env_yaml) + ".bak"
         self._logger.debug("Populating system tests env.yml", filepath=filepath)
-        self._run_command(
-            "cat > ",
-            workdir=".",
-            args=[filepath],
-            stdin=contents,
-            local=True,
-        )
+
+        # if filepath exists, backup the file first (to avoid overriding it)
+        if os.path.isfile(filepath) and not os.path.isfile(backup_filepath):
+            self._logger.debug(
+                "Backing up existing env.yml", destination=backup_filepath
+            )
+            shutil.copy(filepath, backup_filepath)
+
+        serialized_env_config = self._serialize_env_config()
+        with open(filepath, "w") as f:
+            f.write(serialized_env_config)
 
     def _override_mlrun_api_env(self):
         version_specifier = (
@@ -595,14 +602,25 @@ def _run_kubectl_command(self, args, verbose=True):
             verbose=verbose,
         )
 
+    def _serialize_env_config(self, allow_none_values: bool = False):
+        env_config = self._env_config.copy()
+
+        # we sanitize None values from config to avoid "null" values in yaml
+        if not allow_none_values:
+            for key in list(env_config):
+                if env_config[key] is None:
+                    del env_config[key]
+
+        return yaml.safe_dump(env_config)
+
 
 @click.group()
 def main():
     pass
 
 
 @main.command(context_settings=dict(ignore_unknown_options=True))
-@click.argument("mlrun-version", type=str, required=True)
+@click.option("--mlrun-version")
 @click.option(
     "--override-image-registry",
     "-oireg",
@@ -627,25 +645,24 @@ def main():
     default=None,
     help="The commit (in mlrun/mlrun) of the tested mlrun version.",
 )
-@click.argument("data-cluster-ip", type=str, required=True)
-@click.argument("data-cluster-ssh-username", type=str, required=True)
-@click.argument("data-cluster-ssh-password", type=str, required=True)
-@click.argument("app-cluster-ssh-password", type=str, required=True)
-@click.argument("github-access-token", type=str, required=True)
-@click.argument("provctl-download-url", type=str, required=True)
-@click.argument("provctl-download-s3-access-key", type=str, required=True)
-@click.argument("provctl-download-s3-key-id", type=str, required=True)
-@click.argument("mlrun-dbpath", type=str, required=True)
-@click.argument("webapi-direct-url", type=str, required=True)
-@click.argument("framesd-url", type=str, required=True)
-@click.argument("username", type=str, required=True)
-@click.argument("access-key", type=str, required=True)
-@click.argument("iguazio-version", type=str, default=None, required=True)
-@click.argument("spark-service", type=str, required=True)
-@click.argument("password", type=str, default=None, required=False)
-@click.argument("slack-webhook-url", type=str, default=None, required=False)
-@click.argument("mysql-user", type=str, default=None, required=False)
-@click.argument("mysql-password", type=str, default=None, required=False)
+@click.option("--data-cluster-ip", required=True)
+@click.option("--data-cluster-ssh-username", required=True)
+@click.option("--data-cluster-ssh-password", required=True)
+@click.option("--app-cluster-ssh-password", required=True)
+@click.option("--github-access-token", required=True)
+@click.option("--provctl-download-url", required=True)
+@click.option("--provctl-download-s3-access-key", required=True)
+@click.option("--provctl-download-s3-key-id", required=True)
+@click.option("--mlrun-dbpath", required=True)
+@click.option("--webapi-direct-url", required=True)
+@click.option("--framesd-url", required=True)
+@click.option("--username", required=True)
+@click.option("--access-key", required=True)
+@click.option("--iguazio-version", default=None)
+@click.option("--spark-service", required=True)
+@click.option("--slack-webhook-url")
+@click.option("--mysql-user")
+@click.option("--mysql-password")
 @click.option("--purge-db", "-pdb", is_flag=True, help="Purge mlrun db")
 @click.option(
     "--debug",
@@ -674,7 +691,6 @@ def run(
     access_key: str,
     iguazio_version: str,
     spark_service: str,
-    password: str,
     slack_webhook_url: str,
     mysql_user: str,
     mysql_password: str,
@@ -702,7 +718,6 @@ def run(
         access_key,
         iguazio_version,
         spark_service,
-        password,
         slack_webhook_url,
         mysql_user,
         mysql_password,
@@ -717,30 +732,33 @@ def run(
 
 
 @main.command(context_settings=dict(ignore_unknown_options=True))
-@click.argument("mlrun-dbpath", type=str, required=True)
-@click.argument("webapi-direct-url", type=str, required=True)
-@click.argument("framesd-url", type=str, required=True)
-@click.argument("username", type=str, required=True)
-@click.argument("access-key", type=str, required=True)
-@click.argument("spark-service", type=str, required=True)
-@click.argument("password", type=str, default=None, required=False)
-@click.argument("slack-webhook-url", type=str, default=None, required=False)
+@click.option("--mlrun-dbpath", help="The mlrun api address", required=True)
+@click.option("--webapi-direct-url", help="Iguazio webapi direct url")
+@click.option("--framesd-url", help="Iguazio framesd url")
+@click.option("--username", help="Iguazio running username")
+@click.option("--access-key", help="Iguazio running user access key")
+@click.option("--spark-service", help="Iguazio kubernetes spark service name")
+@click.option(
+    "--slack-webhook-url", help="Slack webhook url to send tests notifications to"
+)
 @click.option(
     "--debug",
     "-d",
     is_flag=True,
     help="Don't run the ci only show the commands that will be run",
 )
-@click.argument("branch", type=str, default=None, required=False)
-@click.argument("github-access-token", type=str, default=None, required=False)
+@click.option("--branch", help="The mlrun branch to run the tests against")
+@click.option(
+    "--github-access-token",
+    help="Github access token to use for fetching private functions",
+)
 def env(
     mlrun_dbpath: str,
     webapi_direct_url: str,
     framesd_url: str,
     username: str,
     access_key: str,
     spark_service: str,
-    password: str,
     slack_webhook_url: str,
     debug: bool,
     branch: str,
@@ -753,7 +771,6 @@ def env(
         username=username,
         access_key=access_key,
         spark_service=spark_service,
-        password=password,
         debug=debug,
         slack_webhook_url=slack_webhook_url,
         branch=branch,