ci: parallel system tests [backport 2.5]

.github/workflows/system-tests.yml

@@ -24,7 +24,7 @@ jobs:
           python -c "import os,sys,fnmatch;sys.exit(not bool([_ for pattern in {'ddtrace/*', 'setup*', 'pyproject.toml', '.github/workflows/system-tests.yml'} for _ in fnmatch.filter(os.environ['PATHS'].splitlines(), pattern)]))"
         continue-on-error: true
 
-  system-tests:
+  system-tests-build:
     runs-on: ubuntu-latest
     needs: needs-run
     strategy:
@@ -34,7 +34,7 @@ jobs:
           - weblog-variant: uwsgi-poc
           - weblog-variant: django-poc
           - weblog-variant: fastapi
-        #   runs django-poc for 3.12
+          # runs django-poc for 3.12
           - weblog-variant: python3.12
       fail-fast: false
     env:
@@ -47,6 +47,7 @@ jobs:
       CMAKE_BUILD_PARALLEL_LEVEL: 12
     steps:
       - name: Setup python 3.9
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
         uses: actions/setup-python@v4
         with:
           python-version: '3.9'
@@ -68,95 +69,165 @@ jobs:
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
         run: ./build.sh
 
-      - name: Run
+      - name: Save
+        id: save
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh
+        run: |
+          docker image save system_tests/weblog:latest | gzip > ${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz
+          docker image save system_tests/agent:latest | gzip > ${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz
 
-      - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
+      - uses: actions/upload-artifact@master
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        with:
+          name: ${{ matrix.weblog-variant }}_${{ github.sha }}
+          path: |
+            ${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz
+            ${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz
+            venv
+          retention-days: 2
+
+  system-tests:
+    runs-on: ubuntu-latest
+    needs: [needs-run, system-tests-build]
+    strategy:
+      matrix:
+        weblog-variant: [flask-poc, uwsgi-poc , django-poc, fastapi, python3.12]
+        scenario: [remote-config, appsec, appsec-1, other]
+
+      fail-fast: false
+    env:
+      TEST_LIBRARY: python
+      WEBLOG_VARIANT: ${{ matrix.weblog-variant }}
+      # system-tests requires an API_KEY, but it does not have to be a valid key, as long as we don't run a scenario
+      # that make assertion on backend data. Using a fake key allow to run system tests on PR originating from forks.
+      # If ever it's needed, a valid key exists in the repo, using ${{ secrets.DD_API_KEY }}
+      DD_API_KEY: 1234567890abcdef1234567890abcdef
+      CMAKE_BUILD_PARALLEL_LEVEL: 12
+    steps:
+      - name: Setup python 3.9
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.9'
+
+      - name: Checkout system tests
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        uses: actions/checkout@v3
+        with:
+          repository: 'DataDog/system-tests'
+
+      - uses: actions/download-artifact@master
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        with:
+          name: ${{ matrix.weblog-variant }}_${{ github.sha }}
+          path: ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz
+
+      - name: docker load
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        run: |
+          docker load < ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz
+          docker load < ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz
+
+      - name: move venv
+        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        run: |
+          mv ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/venv venv
+          chmod -R +x venv/bin/*
+
+      - name: Run DEFAULT
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other'
+        run: ./run.sh DEFAULT
+
+      - name: Run SAMPLING
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other'
+        run: ./run.sh SAMPLING
+
+      - name: Run INTEGRATIONS
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other'
+        run: ./run.sh INTEGRATIONS
+
+      - name: Run CROSSED_TRACING_LIBRARIES
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other'
+        run: ./run.sh CROSSED_TRACING_LIBRARIES
+
+      - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config'
         run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES
 
       - name: Run REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config'
         run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING
 
       - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config'
         run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD
 
       - name: Run APPSEC_MISSING_RULES
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_MISSING_RULES
 
       - name: Run APPSEC_CUSTOM_RULES
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_CUSTOM_RULES
 
       - name: Run APPSEC_CORRUPTED_RULES
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_CORRUPTED_RULES
 
       - name: Run APPSEC_RULES_MONITORING_WITH_ERRORS
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_RULES_MONITORING_WITH_ERRORS
 
-      - name: Run APPSEC_BLOCKING
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh APPSEC_BLOCKING
-
-      - name: Run APPSEC_DISABLED
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh APPSEC_DISABLED
-
       - name: Run APPSEC_LOW_WAF_TIMEOUT
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_LOW_WAF_TIMEOUT
 
       - name: Run APPSEC_CUSTOM_OBFUSCATION
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_CUSTOM_OBFUSCATION
 
       - name: Run APPSEC_RATE_LIMITER
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec'
         run: ./run.sh APPSEC_RATE_LIMITER
 
-      - name: Run APPSEC_BLOCKING_FULL_DENYLIST
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh APPSEC_BLOCKING_FULL_DENYLIST
-
-      - name: Run APPSEC_REQUEST_BLOCKING
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh APPSEC_REQUEST_BLOCKING
-
       - name: Run APPSEC_RUNTIME_ACTIVATION
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
         run: ./run.sh APPSEC_RUNTIME_ACTIVATION
 
       - name: Run APPSEC_WAF_TELEMETRY
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
         run: ./run.sh APPSEC_WAF_TELEMETRY
 
-      - name: Run SAMPLING
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh SAMPLING
+      - name: Run APPSEC_DISABLED
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
+        run: ./run.sh APPSEC_DISABLED
 
-      - name: Run INTEGRATIONS 
-        if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
-        run: ./run.sh INTEGRATIONS 
+      - name: Run APPSEC_BLOCKING
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
+
+        run: ./run.sh APPSEC_BLOCKING
+      - name: Run APPSEC_BLOCKING_FULL_DENYLIST
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
+        run: ./run.sh APPSEC_BLOCKING_FULL_DENYLIST
+
+      - name: Run APPSEC_REQUEST_BLOCKING
+        if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1'
+        run: ./run.sh APPSEC_REQUEST_BLOCKING
 
-      # even on failures, we want to have artifact to be able to investigate
       # The compress step speed up a lot the upload artifact process
       - name: Compress artifact
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
+        id: compress-artifact
         run: tar -czvf artifact.tar.gz $(ls | grep logs)
 
       - name: Upload artifact
         uses: actions/upload-artifact@v3
         if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule'
         with:
-          name: logs_${{ matrix.weblog-variant }}
+          name: logs_${{ matrix.weblog-variant }}_${{ matrix.scenario }}
           path: artifact.tar.gz
 
+
   parametric:
     runs-on: ubuntu-latest
     needs: needs-run
@@ -199,3 +270,4 @@ jobs:
         with:
           name: logs_parametric
           path: artifact.tar.gz
+