.github/workflows/detection-testing.yml #18184
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: detection-testing | |
| on: | |
| push: | |
| pull_request: | |
| types: [opened, reopened] | |
| schedule: | |
| - cron: "44 4 * * *" | |
| jobs: | |
| validate-tag-if-present: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: TAGGED, Validate that the tag is in the correct format | |
| run: | | |
| echo "The GITHUB_REF: $GITHUB_REF" | |
| #First check to see if the release is a tag | |
| if [[ $GITHUB_REF =~ refs/tags/* ]]; then | |
| #Yes, this is a tag, so we need to test to make sure that the tag | |
| #is in the correct format (like v1.10.20) | |
| if [[ $GITHUB_REF =~ refs/tags/v[0-9]+.[0-9]+.[0-9]+ ]]; then | |
| echo "PASS: Tagged release with good format" | |
| exit 0 | |
| else | |
| echo "FAIL: Tagged release with bad format" | |
| exit 1 | |
| fi | |
| else | |
| echo "PASS: Not a tagged release" | |
| exit 0 | |
| fi | |
| quit-for-dependabot: | |
| runs-on: ubuntu-latest | |
| if: github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' | |
| steps: | |
| - name: "Placeholder" | |
| run: | | |
| echo "yes it ran" | |
| docker-detection-testing-setup: | |
| runs-on: ubuntu-latest | |
| if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job | |
| needs: [validate-tag-if-present, quit-for-dependabot] | |
| steps: | |
| - name: Get branch and PR required for detection testing main.py | |
| id: vars | |
| run: | | |
| echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" | |
| - name: Checkout Repo | |
| uses: actions/checkout@v2 | |
| #with: | |
| # ref: develop | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy | |
| architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified | |
| cache: 'pip' | |
| - name: Install Python Dependencies | |
| run: | | |
| python -m venv .venv | |
| source .venv/bin/activate | |
| python -m pip install wheel | |
| python -m pip install -r requirements.txt | |
| - name: Run the CI | |
| run: | | |
| source .venv/bin/activate | |
| cd bin/docker_detection_tester | |
| echo "github.event.issue.pull_request : [${{ github.event.issue.pull_request }}]" | |
| echo "github.event.pull_request.number : [${{ github.event.pull_request.number }}]" | |
| echo "steps.vars.outputs.branch : [${{ steps.vars.outputs.branch }}]" | |
| echo "github.event.pull_request.head.ref : [${{ github.event.pull_request.head.ref }}]" | |
| echo "github.event_name : [${{ github.event_name }}]" | |
| if [[ ${{ github.event_name }} == schedule ]]; then | |
| # Note that scheduled actions ONLY run on the default branch, so it won't run on all other branches! | |
| echo "Running a nightly test on all detections OR a commit was made directly to develop" | |
| python detection_testing_execution.py run --branch develop --mode all --mock --config_file test_config_github_actions.json | |
| elif [[ ! -z "${{ github.event.pull_request.head.ref }}" && ! -z "${{ github.event.pull_request.number }}" ]]; then | |
| echo "Pull request from source branch [${{ github.event.pull_request.head.ref }}] for PR number [${{ github.event.issue.number }}]" | |
| python detection_testing_execution.py run --branch ${{ github.event.pull_request.head.ref }} --pr_number ${{ github.event.pull_request.number }} --mode changes --mock --config_file test_config_github_actions.json | |
| else | |
| echo "Push from branch [${{ steps.vars.outputs.branch }}]" | |
| python detection_testing_execution.py run --branch ${{ steps.vars.outputs.branch }} --mode changes --mock --config_file test_config_github_actions.json | |
| fi | |
| mv *-test-run.json replicate_test.json | |
| - name: Upload Test Results Files | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: testing-results-config | |
| path: | | |
| bin/docker_detection_tester/prior_config/apps/DA-ESS-ContentUpdate-latest.tar.gz | |
| bin/docker_detection_tester/prior_config/config_tests_0.json | |
| bin/docker_detection_tester/prior_config/config_tests_1.json | |
| bin/docker_detection_tester/prior_config/config_tests_2.json | |
| bin/docker_detection_tester/prior_config/config_tests_3.json | |
| bin/docker_detection_tester/prior_config/config_tests_4.json | |
| bin/docker_detection_tester/prior_config/config_tests_5.json | |
| bin/docker_detection_tester/prior_config/config_tests_6.json | |
| bin/docker_detection_tester/prior_config/config_tests_7.json | |
| bin/docker_detection_tester/prior_config/config_tests_8.json | |
| bin/docker_detection_tester/prior_config/config_tests_9.json | |
| - name: Upload File to Enable Replication of the Test at a Different Time or Place | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: replicate_test | |
| path: | | |
| bin/docker_detection_tester/replicate_test.json | |
| docker-detection-testing-execution: | |
| runs-on: ubuntu-latest | |
| if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job | |
| needs: [docker-detection-testing-setup] | |
| strategy: | |
| matrix: | |
| manifest_filename: ["config_tests_0.json", | |
| "config_tests_1.json", | |
| "config_tests_2.json", | |
| "config_tests_3.json", | |
| "config_tests_4.json", | |
| "config_tests_5.json", | |
| "config_tests_6.json", | |
| "config_tests_7.json", | |
| "config_tests_8.json", | |
| "config_tests_9.json"] | |
| steps: | |
| - name: Get branch and PR required for detection testing main.py | |
| id: vars | |
| run: | | |
| echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" | |
| - name: Checkout Repo | |
| uses: actions/checkout@v2 | |
| #with: | |
| # ref: develop | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: testing-results-config | |
| path: bin/docker_detection_tester/prior_config | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy | |
| architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified | |
| cache: 'pip' | |
| - name: Install Python Dependencies | |
| run: | | |
| python -m venv .venv | |
| source .venv/bin/activate | |
| python -m pip install wheel | |
| python -m pip install -r requirements.txt | |
| - name: Run the CI | |
| run: | | |
| source .venv/bin/activate | |
| cd bin/docker_detection_tester | |
| python detection_testing_execution.py run -c prior_config/${{ matrix.manifest_filename}} | |
| - name: Upload Test Results Files | |
| uses: actions/upload-artifact@v2 | |
| with: | |
| name: ${{ matrix.manifest_filename}}.results | |
| path: | | |
| bin/docker_detection_tester/test_results/success.csv | |
| bin/docker_detection_tester/test_results/error.csv | |
| bin/docker_detection_tester/test_results/failure.csv | |
| bin/docker_detection_tester/test_results/combined.csv | |
| bin/docker_detection_tester/test_results/success.json | |
| bin/docker_detection_tester/test_results/error.json | |
| bin/docker_detection_tester/test_results/failure.json | |
| bin/docker_detection_tester/test_results/combined.json | |
| bin/docker_detection_tester/test_results/summary.json | |
| docker-detection-testing-execution-merge-results: | |
| runs-on: ubuntu-latest | |
| if: "!contains(github.ref, 'refs/tags/')" #don't run on tags - future steps won't run either since they depend on this job | |
| needs: [docker-detection-testing-setup, docker-detection-testing-execution] | |
| steps: | |
| - name: Get branch and PR required for detection testing main.py | |
| id: vars | |
| run: | | |
| echo "::set-output name=branch::${GITHUB_REF#refs/heads/}" | |
| - name: Checkout Repo | |
| uses: actions/checkout@v2 | |
| #with: | |
| # ref: develop | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_0.json.results | |
| path: bin/docker_detection_tester/results_0 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_1.json.results | |
| path: bin/docker_detection_tester/results_1 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_2.json.results | |
| path: bin/docker_detection_tester/results_2 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_3.json.results | |
| path: bin/docker_detection_tester/results_3 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_4.json.results | |
| path: bin/docker_detection_tester/results_4 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_5.json.results | |
| path: bin/docker_detection_tester/results_5 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_6.json.results | |
| path: bin/docker_detection_tester/results_6 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_7.json.results | |
| path: bin/docker_detection_tester/results_7 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_8.json.results | |
| path: bin/docker_detection_tester/results_8 | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v2 | |
| with: | |
| name: config_tests_9.json.results | |
| path: bin/docker_detection_tester/results_9 | |
| - uses: actions/setup-python@v2 | |
| with: | |
| python-version: '3.9' #Available versions here - https://github.com/actions/python-versions/releases easy to change/make a matrix/use pypy | |
| architecture: 'x64' # optional x64 or x86. Defaults to x64 if not specified | |
| cache: 'pip' | |
| - name: Install Python Dependencies | |
| run: | | |
| python -m venv .venv | |
| source .venv/bin/activate | |
| python -m pip install wheel | |
| python -m pip install -r requirements.txt | |
| - name: Merge Detections into single File | |
| run: | | |
| source .venv/bin/activate | |
| cd bin/docker_detection_tester | |
| python summarize_json.py --files results_*/summary.json --output_filename summary_test_results.json | |
| - name: Upload Summary Test Results JSON | |
| uses: actions/upload-artifact@v2 | |
| if: always() | |
| with: | |
| name: SummaryTestResults | |
| path: | | |
| bin/docker_detection_tester/summary_test_results.json | |
| - name: Upload Failures Manifest on Failure | |
| uses: actions/upload-artifact@v2 | |
| if: failure() | |
| with: | |
| name: DetectionFailureManifest | |
| path: | | |
| bin/docker_detection_tester/detection_failure_manifest.json | |
| #Always clean these up, they make the output messy | |
| - name: Clean up intermediate Files | |
| uses: geekyeggo/delete-artifact@v1 | |
| if: always() | |
| with: | |
| name: | | |
| config_tests_0.json.results | |
| config_tests_1.json.results | |
| config_tests_2.json.results | |
| config_tests_3.json.results | |
| config_tests_4.json.results | |
| config_tests_5.json.results | |
| config_tests_6.json.results | |
| config_tests_7.json.results | |
| config_tests_8.json.results | |
| config_tests_9.json.results | |
| - name: Log in to S3 for Artifact Uploads | |
| if: ${{ github.event_name == 'schedule' }} | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-west-2 | |
| - name: Upload S3 Badge and Summary Artifacts for Nightly Scheduled Run | |
| if: ${{ github.event_name == 'schedule' }} | |
| run: | | |
| cd bin/docker_detection_tester | |
| python generate_detection_coverage_badge.py --input_summary_file summary_test_results.json --output_badge_file detection_coverage.svg --badge_string "Pass Rate" | |
| #Upload artifact (summary test results) | |
| aws s3 cp summary_test_results.json s3://security-content/reporting/summary_test_results.json | |
| #Since these reside in a public bucket, no need to explicitly mark as public | |
| # make the file public since it is not by default | |
| #aws s3api put-object-acl --bucket security-content --key reporting/summary_test_results.json --acl public-read | |
| #Upload artifact (test results coverage badge) | |
| aws s3 cp detection_coverage.svg s3://security-content/reporting/detection_coverage.svg | |
| #Since these reside in a public bucket, no need to explicitly mark as public | |
| # make the file public since it is not by default | |
| #aws s3api put-object-acl --bucket security-content --key reporting/detection_coverage.svg --acl public-read | |