[System Tests] Fix not continuing to run when one job in matrix fails (…

…#2209)
mlrun · Aug 3, 2022 · ed54118 · ed54118
1 parent 1e59e07
commit ed54118
Showing 1 changed file with 9 additions and 24 deletions.
diff --git a/.github/workflows/system-tests-enterprise.yml b/.github/workflows/system-tests-enterprise.yml
@@ -8,8 +8,8 @@ on:
   schedule:
 
     # * is a special character in YAML so you have to quote this string
-    # Run the system tests every 7 hours
-    - cron:  '0 */7 * * *'
+    # Run the system tests every 8 hours (cron hours should divide 24 equally, otherwise there will be an overlap at the end of the day)
+    - cron:  '0 */8 * * *'
 
   workflow_dispatch:
     inputs:
@@ -37,6 +37,7 @@ on:
         description: 'Override the configured target system iguazio version (leave empty to resolve automatically by the mlrun version)'
         required: false
 
+concurrency: one-at-a-time
 jobs:
   prepare-system-tests-enterprise-ci:
     # When increasing the timeout make sure it's not larger than the schedule cron interval
@@ -122,17 +123,6 @@ jobs:
           override_iguazio_version=${{ github.event.inputs.override_iguazio_version }} && \
           iguazio_system_version=`echo "3.4.2-b149.20220705184806"` && \
           resolved_iguazio_version=${override_iguazio_version:-$iguazio_system_version} && echo $resolved_iguazio_version)"
-    - name: Wait for existing runs to complete
-      uses: softprops/turnstyle@v1
-      with:
-        poll-interval-seconds: 20s
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    - name: Write google big query credentials to file
-      id: credentials
-      run: |
-        export google_big_query_credentials_json_path="/home/runner/work/mlrun/mlrun/tests/system/google-big-query-credentials.json"
-        echo  ${{ secrets.LATEST_MLRUN_SYSTEM_TESTS_GOOGLE_BIG_QUERY_CREDENTIALS_JSON }} > "$google_big_query_credentials_json_path"
     - name: Prepare System Test env.yaml and MLRun installation from current branch
       timeout-minutes: 45
       run: |
@@ -161,15 +151,16 @@ jobs:
     # When increasing the timeout make sure it's not larger than the schedule cron interval
     timeout-minutes: 360
     name: Run System Tests Enterprise
-    runs-on: ubuntu-latest
+    # requires prepare to finish before starting
     needs: [prepare-system-tests-enterprise-ci]
+    runs-on: ubuntu-latest
+    # let's not run this on every fork, change to your fork when developing
+    if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch'
     strategy:
+      fail-fast: false
       max-parallel: 1
       matrix:
         test_component: [api,runtimes,feature_store,projects,model_monitoring,examples,demos,backward_compatibility]
-    # let's not run this on every fork, change to your fork when developing
-    if: github.repository == 'mlrun/mlrun' || github.event_name == 'workflow_dispatch'
-
     steps:
     - uses: actions/checkout@v2
     - name: Set up python 3.7
@@ -245,11 +236,6 @@ jobs:
           override_iguazio_version=${{ github.event.inputs.override_iguazio_version }} && \
           iguazio_system_version=`echo "3.4.2-b149.20220705184806"` && \
           resolved_iguazio_version=${override_iguazio_version:-$iguazio_system_version} && echo $resolved_iguazio_version)"
-    - name: Write google big query credentials to file
-      id: credentials
-      run: |
-        export google_big_query_credentials_json_path="/home/runner/work/mlrun/mlrun/tests/system/google-big-query-credentials.json"
-        echo  ${{ secrets.LATEST_MLRUN_SYSTEM_TESTS_GOOGLE_BIG_QUERY_CREDENTIALS_JSON }} > "$google_big_query_credentials_json_path"
     - name: Prepare System Test env.yaml and MLRun installation from current branch
       timeout-minutes: 5
       run: |
@@ -261,10 +247,9 @@ jobs:
           "${{ secrets.LATEST_SYSTEM_TEST_ACCESS_KEY }}" \
           "${{ secrets.LATEST_SYSTEM_TEST_SPARK_SERVICE }}" \
           "${{ secrets.LATEST_SYSTEM_TEST_PASSWORD }}" \
-
     - name: Run System Tests
       run: |
         MLRUN_SYSTEM_TESTS_CLEAN_RESOURCES="${{ steps.computed_params.outputs.mlrun_system_tests_clean_resources }}" \
         MLRUN_VERSION="${{ steps.computed_params.outputs.mlrun_version }}" \
         MLRUN_SYSTEM_TESTS_COMPONENT="${{ matrix.test_component }}" \
-          make test-system-dockerized
+          make test-system-dockerized