System Test CI (#444)

.github/workflows/system-tests.yml

@@ -0,0 +1,81 @@
+name: System Tests
+
+on:
+  push:
+    branches:
+      - '.+-system-tests'
+
+  schedule:
+
+    # * is a special character in YAML so you have to quote this string
+    # Run the system tests every 3 hours
+    - cron:  '0 */3 * * *'
+
+jobs:
+  run-system-tests-ci:
+    timeout-minutes: 60
+    name: Run System Tests
+    runs-on: ubuntu-latest
+
+    # let's not run this on every fork, change to your fork when developing
+    if: github.repository == 'mlrun/mlrun'
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Install automation scripts dependencies and add mlrun to dev packages
+      run: pip install -r automation/requirements.txt && python setup.py develop
+    - name: Install curl and jq
+      run: sudo apt-get install curl jq
+    - name: Extract git hashes and latest version
+      id: git_info
+      run: |
+        echo "::set-output name=mlrun_hash::$( \
+          cd /tmp && \
+          git clone --single-branch --branch development https://github.com/mlrun/mlrun.git mlrun-upstream 2> /dev/null && \
+          cd mlrun-upstream && \
+          git rev-parse --short HEAD && \
+          cd .. && \
+          rm -rf mlrun-upstream)"
+        echo "::set-output name=ui_hash::$( \
+          cd /tmp && \
+          git clone --single-branch --branch development https://github.com/mlrun/ui.git mlrun-ui 2> /dev/null && \
+          cd mlrun-ui && \
+          git rev-parse --short HEAD && \
+          cd .. && \
+          rm -rf mlrun-ui)"
+        echo "::set-output name=latest_version::$(curl -sf https://pypi.org/pypi/mlrun/json | jq -r '.info.version')"
+    - name: Set computed versions params
+      id: mlrun_versions
+      run: |
+        echo "::set-output name=mlrun::${{ steps.git_info.outputs.latest_version }}-${{ steps.git_info.outputs.mlrun_hash }}"
+        echo "::set-output name=ui::${{ steps.git_info.outputs.latest_version }}-${{ steps.git_info.outputs.ui_hash }}"
+    - name: Wait for existing runs to complete
+      uses: softprops/turnstyle@v1
+      with:
+        poll-interval-seconds: 20s
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - name: Prepare System Test env.yaml and system
+      run: |
+        python automation/system_test/prepare.py run \
+          ${{ steps.mlrun_versions.outputs.mlrun }} \
+          ${{ secrets.SYSTEM_TEST_DATA_CLUSTER_IP }} \
+          ${{ secrets.SYSTEM_TEST_DATA_CLUSTER_SSH_PASSWORD }} \
+          ${{ secrets.SYSTEM_TEST_APP_CLUSTER_SSH_PASSWORD }} \
+          ${{ secrets.SYSTEM_TEST_GITHUB_ACCESS_TOKEN }} \
+          ${{ secrets.SYSTEM_TEST_MLRUN_DB_PATH }} \
+          ${{ secrets.SYSTEM_TEST_WEBAPI_DIRECT_URL }} \
+          ${{ secrets.SYSTEM_TEST_USERNAME }} \
+          ${{ secrets.SYSTEM_TEST_ACCESS_KEY }} \
+          ${{ secrets.SYSTEM_TEST_PASSWORD }} \
+          --override-image-registry "ghcr.io/" \
+          --override-image-repo mlrun \
+          --override-mlrun-images \
+          "ghcr.io/mlrun/mlrun-api:${{ steps.mlrun_versions.outputs.mlrun }},ghcr.io/mlrun/mlrun-ui:${{ steps.mlrun_versions.outputs.ui }}"
+
+    - name: Run System Tests
+      run: make test-system

Makefile

@@ -385,6 +385,7 @@ test-dockerized: build-test ## Run mlrun tests in docker container
 .PHONY: test
 test: clean ## Run mlrun tests
 	python -m pytest -v \
+		--capture=no \
 		--disable-warnings \
 		-rf \
 		tests

automation/requirements.txt

@@ -1,2 +1,3 @@
 # ==7.0 from kfp
-click==7.0
+click==7.0
+paramiko~=2.7