From ea63a3ac922f1a03d03f46fde461b863e398097b Mon Sep 17 00:00:00 2001 From: Federico Mon Date: Mon, 1 Apr 2024 19:13:21 +0200 Subject: [PATCH] ci: parallel system tests (#8813) CI: Introduces concurrency to System Tests execution in order to decrease wall time. Decreases wall time for system tests from ~37 min to ~23 min. - [x] Change(s) are motivated and described in the PR description - [x] Testing strategy is described if automated tests are not included in the PR - [x] Risks are described (performance impact, potential for breakage, maintainability) - [x] Change is maintainable (easy to change, telemetry, documentation) - [x] [Library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) are followed or label `changelog/no-changelog` is set - [x] Documentation is included (in-code, generated user docs, [public corp docs](https://github.com/DataDog/documentation/)) - [x] Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) - [x] If this PR changes the public interface, I've notified `@DataDog/apm-tees`. - [x] If change touches code that signs or publishes builds or packages, or handles credentials of any kind, I've requested a review from `@DataDog/security-design-and-guidance`. - [x] Title is accurate - [x] All changes are related to the pull request's stated goal - [x] Description motivates each change - [x] Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - [x] Testing strategy adequately addresses listed risks - [x] Change is maintainable (easy to change, telemetry, documentation) - [x] Release note makes sense to a user of the library - [x] Author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - [x] Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting) --------- Co-authored-by: Emmett Butler <723615+emmettbutler@users.noreply.github.com> (cherry picked from commit ad2fe2554a53d6bbee12a0f5a97f44c54f27b508) --- .github/workflows/system-tests.yml | 152 +++++++++++++++++++++-------- 1 file changed, 112 insertions(+), 40 deletions(-) diff --git a/.github/workflows/system-tests.yml b/.github/workflows/system-tests.yml index b188caf558..797d74b2e0 100644 --- a/.github/workflows/system-tests.yml +++ b/.github/workflows/system-tests.yml @@ -24,7 +24,7 @@ jobs: python -c "import os,sys,fnmatch;sys.exit(not bool([_ for pattern in {'ddtrace/*', 'setup*', 'pyproject.toml', '.github/workflows/system-tests.yml'} for _ in fnmatch.filter(os.environ['PATHS'].splitlines(), pattern)]))" continue-on-error: true - system-tests: + system-tests-build: runs-on: ubuntu-latest needs: needs-run strategy: @@ -34,7 +34,7 @@ jobs: - weblog-variant: uwsgi-poc - weblog-variant: django-poc - weblog-variant: fastapi - # runs django-poc for 3.12 + # runs django-poc for 3.12 - weblog-variant: python3.12 fail-fast: false env: @@ -47,6 +47,7 @@ jobs: CMAKE_BUILD_PARALLEL_LEVEL: 12 steps: - name: Setup python 3.9 + if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' uses: actions/setup-python@v4 with: python-version: '3.9' @@ -68,95 +69,165 @@ jobs: if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' run: ./build.sh - - name: Run + - name: Save + id: save if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' - run: ./run.sh + run: | + docker image save system_tests/weblog:latest | gzip > ${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz + docker image save system_tests/agent:latest | gzip > ${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz - - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES + - uses: actions/upload-artifact@master + if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + with: + name: ${{ matrix.weblog-variant }}_${{ github.sha }} + path: | + ${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz + ${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz + venv + retention-days: 2 + + system-tests: + runs-on: ubuntu-latest + needs: [needs-run, system-tests-build] + strategy: + matrix: + weblog-variant: [flask-poc, uwsgi-poc , django-poc, fastapi, python3.12] + scenario: [remote-config, appsec, appsec-1, other] + + fail-fast: false + env: + TEST_LIBRARY: python + WEBLOG_VARIANT: ${{ matrix.weblog-variant }} + # system-tests requires an API_KEY, but it does not have to be a valid key, as long as we don't run a scenario + # that make assertion on backend data. Using a fake key allow to run system tests on PR originating from forks. + # If ever it's needed, a valid key exists in the repo, using ${{ secrets.DD_API_KEY }} + DD_API_KEY: 1234567890abcdef1234567890abcdef + CMAKE_BUILD_PARALLEL_LEVEL: 12 + steps: + - name: Setup python 3.9 if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Checkout system tests + if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + uses: actions/checkout@v3 + with: + repository: 'DataDog/system-tests' + + - uses: actions/download-artifact@master + if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + with: + name: ${{ matrix.weblog-variant }}_${{ github.sha }} + path: ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz + + - name: docker load + if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + run: | + docker load < ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/${{ matrix.weblog-variant}}_weblog_${{ github.sha }}.tar.gz + docker load < ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/${{ matrix.weblog-variant}}_agent_${{ github.sha }}.tar.gz + + - name: move venv + if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + run: | + mv ${{ matrix.weblog-variant}}_${{ github.sha }}.tar.gz/venv venv + chmod -R +x venv/bin/* + + - name: Run DEFAULT + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' + run: ./run.sh DEFAULT + + - name: Run SAMPLING + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' + run: ./run.sh SAMPLING + + - name: Run INTEGRATIONS + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' + run: ./run.sh INTEGRATIONS + + - name: Run CROSSED_TRACING_LIBRARIES + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'other' + run: ./run.sh CROSSED_TRACING_LIBRARIES + + - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config' run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_FEATURES - name: Run REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config' run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_LIVE_DEBUGGING - name: Run REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'remote-config' run: ./run.sh REMOTE_CONFIG_MOCKED_BACKEND_ASM_DD - name: Run APPSEC_MISSING_RULES - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_MISSING_RULES - name: Run APPSEC_CUSTOM_RULES - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_CUSTOM_RULES - name: Run APPSEC_CORRUPTED_RULES - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_CORRUPTED_RULES - name: Run APPSEC_RULES_MONITORING_WITH_ERRORS - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_RULES_MONITORING_WITH_ERRORS - - name: Run APPSEC_BLOCKING - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' - run: ./run.sh APPSEC_BLOCKING - - - name: Run APPSEC_DISABLED - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' - run: ./run.sh APPSEC_DISABLED - - name: Run APPSEC_LOW_WAF_TIMEOUT - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_LOW_WAF_TIMEOUT - name: Run APPSEC_CUSTOM_OBFUSCATION - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_CUSTOM_OBFUSCATION - name: Run APPSEC_RATE_LIMITER - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec' run: ./run.sh APPSEC_RATE_LIMITER - - name: Run APPSEC_BLOCKING_FULL_DENYLIST - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' - run: ./run.sh APPSEC_BLOCKING_FULL_DENYLIST - - - name: Run APPSEC_REQUEST_BLOCKING - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' - run: ./run.sh APPSEC_REQUEST_BLOCKING - - name: Run APPSEC_RUNTIME_ACTIVATION - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_RUNTIME_ACTIVATION - name: Run APPSEC_WAF_TELEMETRY - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' run: ./run.sh APPSEC_WAF_TELEMETRY - - name: Run SAMPLING - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' - run: ./run.sh SAMPLING + - name: Run APPSEC_DISABLED + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + run: ./run.sh APPSEC_DISABLED - - name: Run INTEGRATIONS - if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' - run: ./run.sh INTEGRATIONS + - name: Run APPSEC_BLOCKING + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + + run: ./run.sh APPSEC_BLOCKING + - name: Run APPSEC_BLOCKING_FULL_DENYLIST + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + run: ./run.sh APPSEC_BLOCKING_FULL_DENYLIST + + - name: Run APPSEC_REQUEST_BLOCKING + if: (needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule') && matrix.scenario == 'appsec-1' + run: ./run.sh APPSEC_REQUEST_BLOCKING - # even on failures, we want to have artifact to be able to investigate # The compress step speed up a lot the upload artifact process - name: Compress artifact if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' + id: compress-artifact run: tar -czvf artifact.tar.gz $(ls | grep logs) - name: Upload artifact uses: actions/upload-artifact@v3 if: needs.needs-run.outputs.outcome == 'success' || github.event_name == 'schedule' with: - name: logs_${{ matrix.weblog-variant }} + name: logs_${{ matrix.weblog-variant }}_${{ matrix.scenario }} path: artifact.tar.gz + parametric: runs-on: ubuntu-latest needs: needs-run @@ -199,3 +270,4 @@ jobs: with: name: logs_parametric path: artifact.tar.gz +