Merge branch 'main' into updatecli_main_updatecli-hermit-main

elastic · Apr 9, 2024 · c91ddb0 · c91ddb0
2 parents 041f53d + 84229c2
commit c91ddb0
Show file tree

Hide file tree

Showing 170 changed files with 8,795 additions and 5,706 deletions.
diff --git a/.github/actions/aws-ci/action.yml b/.github/actions/aws-ci/action.yml
@@ -0,0 +1,69 @@
+name: 'AWS CI'
+description: 'AWS integration tests'
+inputs:
+  elk-version:
+    description: 'ELK version'
+    required: true
+  aws-access-key-id:
+    description: 'AWS access key id'
+    required: true
+  aws-secret-access-key:
+    description: 'AWS secret access key'
+    required: true
+  aws-account-type:
+    description: 'AWS account type'
+    required: false
+    default: single-account
+
+  debug:
+    description: 'debug'
+    required: false
+    default: 'false'
+runs:
+  using: composite
+  steps:
+    - name: Init Integration
+      uses: ./.github/actions/init-integration
+      with:
+        elk-version: ${{ inputs.elk-version }}
+
+    - name: Run cloudbeat in background
+      env:
+        ES_HOST: http://localhost:9200
+        ES_USERNAME: elastic
+        ES_PASSWORD: changeme
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws-access-key-id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws-secret-access-key }}
+        AWS_ACCOUNT_TYPE: ${{ inputs.aws-account-type }}
+      shell: bash
+      run: |
+        ./cloudbeat -c deploy/aws/cloudbeat-aws.yml -d '*' &
+
+    - name: Wait for cloudbeat to send some events
+      shell: bash
+      run: sleep 20
+
+    - name: Check for findings
+      working-directory: ./tests
+      env:
+        USE_K8S: "false"
+      shell: bash
+      run: poetry run pytest -k "aws" --alluredir=./allure/results/ --clean-alluredir
+
+    - name: Upload test results
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: allure-results-ci-aws
+        path: tests/allure/results/
+        overwrite: true
+
+    - if: ${{ failure() || cancelled() || inputs.debug == 'true' }}
+      name: Upload cloudbeat logs
+      uses: actions/upload-artifact@v4
+      with:
+        name: cloubeat-logs-ci-aws
+        path: logs/
+        if-no-files-found: warn
+        retention-days: 1
+        overwrite: true
diff --git a/.github/actions/azure-ci/action.yml b/.github/actions/azure-ci/action.yml
@@ -0,0 +1,70 @@
+name: 'Azure CI'
+description: 'Azure integration tests'
+inputs:
+  elk-version:
+    description: 'ELK version'
+    required: true
+  azure-client-id:
+    description: 'Azure client id'
+    required: true
+  azure-tenant-id:
+    description: 'Azure tenant id'
+    required: true
+  azure-client-secret:
+    description: 'Azure client secret'
+    required: true
+
+  debug:
+    description: 'debug'
+    required: false
+    default: 'false'
+runs:
+  using: composite
+  steps:
+    - name: Init Integration
+      uses: ./.github/actions/init-integration
+      with:
+        elk-version: ${{ inputs.elk-version }}
+
+    - name: Run cloudbeat in background
+      env:
+        ES_HOST: http://localhost:9200
+        ES_USERNAME: elastic
+        ES_PASSWORD: changeme
+        AZURE_ACCOUNT_TYPE: single-account
+        AZURE_CREDENTIALS_TYPE: service_principal_with_client_secret
+        AZURE_CLIENT_ID: ${{ inputs.azure-client-id }}
+        AZURE_TENANT_ID: ${{ inputs.azure-tenant-id }}
+        AZURE_CLIENT_SECRET: ${{ inputs.azure-client-secret }}
+      shell: bash
+      run: |
+        ./cloudbeat -c deploy/azure/cloudbeat-azure.yml -d '*' &
+
+    - name: Wait for cloudbeat to send some events
+      shell: bash
+      run: sleep 15
+
+    - name: Check for findings
+      working-directory: ./tests
+      env:
+        USE_K8S: "false"
+      shell: bash
+      run: poetry run pytest -k "azure" --alluredir=./allure/results/ --clean-alluredir
+
+    - name: Upload test results
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: allure-results-ci-azure
+        path: tests/allure/results/
+        overwrite: true
+
+    - if: ${{ failure() || cancelled() || inputs.debug == 'true' }}
+      name: Upload cloudbeat logs
+      uses: actions/upload-artifact@v4
+      with:
+        name: cloubeat-logs-ci-azure
+        path: logs/
+        if-no-files-found: warn
+        retention-days: 1
+        overwrite: true
diff --git a/.github/actions/cnvm-ci/action.yml b/.github/actions/cnvm-ci/action.yml
@@ -0,0 +1,72 @@
+name: 'CNVM CI'
+description: 'CNVM integration tests'
+inputs:
+  elk-version:
+    description: 'ELK version'
+    required: true
+  aws-access-key-id:
+    description: 'AWS access key id'
+    required: true
+  aws-secret-access-key:
+    description: 'AWS secret access key'
+    required: true
+  aws-region:
+    description: 'AWS region'
+    required: true
+
+  debug:
+    description: 'debug'
+    required: false
+    default: 'false'
+runs:
+  using: composite
+  steps:
+    - name: Init Integration
+      uses: ./.github/actions/init-integration
+      with:
+        elk-version: ${{ inputs.elk-version }}
+
+    - name: Configure AWS credentials
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        aws-access-key-id: ${{ inputs.aws-access-key-id }}
+        aws-secret-access-key: ${{ inputs.aws-secret-access-key }}
+        aws-region: ${{ inputs.aws-region }}
+
+    - name: Run cloudbeat in background
+      env:
+        ES_HOST: http://localhost:9200
+        ES_USERNAME: elastic
+        ES_PASSWORD: changeme
+      shell: bash
+      run: ./cloudbeat -c deploy/vulnerability/cloudbeat-vuln-mgmt.yml -d '*' &
+
+    - name: Wait for cloudbeat to send some events
+      shell: bash
+      run: sleep 10
+
+    - name: Check for findings
+      working-directory: ./tests
+      env:
+        USE_K8S: "false"
+      shell: bash
+      run: |
+        poetry run pytest -k "cnvm" --alluredir=./allure/results/ --clean-alluredir
+
+    - name: Upload test results
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: allure-results-ci-cnvm
+        path: tests/allure/results/
+        overwrite: true
+
+    - if: ${{ failure() || cancelled() || inputs.debug == 'true' }}
+      name: Upload cloudbeat logs
+      uses: actions/upload-artifact@v4
+      with:
+        name: cloubeat-logs-ci-cnvm
+        path: logs/
+        if-no-files-found: warn
+        retention-days: 1
+        overwrite: true
diff --git a/.github/actions/docker-images/action.yml b/.github/actions/docker-images/action.yml
@@ -0,0 +1,139 @@
+name: 'Docker Images'
+description: 'Build docker images'
+inputs:
+  build-docker-images:
+    description: "Build docker images"
+    required: false
+    default: 'true'
+
+  container-image-suffix:
+    description: 'Container image suffix'
+    required: false
+    default: ${{ github.run_id }}
+  elastic-agent-docker-image:
+    description: 'Elastic-Agent docker image'
+    required: false
+    default: "704479110758.dkr.ecr.eu-west-2.amazonaws.com/elastic-agent"
+  elastic-agent-docker-image-tag:
+    description: 'Elastic-Agent docker image tag'
+    required: false
+    default: "8.14.0-SNAPSHOT"
+
+  docker-build-cache-folder:
+    description: 'Docker build cache folder'
+    required: false
+    default: '/tmp/.buildx-cache'
+  docker-images-folder:
+    description: 'Docker build cache folder'
+    required: false
+    default: '/tmp/.docker-images'
+
+  goos:
+    description: 'GOOS environment variable'
+    required: false
+    default: 'linux'
+  goarch:
+    description: 'GOARCH environment variable'
+    required: false
+    default: 'amd64'
+  docker-build-platforms:
+    description: 'Docker build platforms'
+    required: false
+    default: 'linux/amd64'
+runs:
+  using: composite
+  steps:
+    - name: Init directories
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.docker-build-cache-folder }}/{cloudbeat,agent,pytest}
+        mkdir -p ${{ inputs.docker-build-cache-folder }}-new/{cloudbeat,agent,pytest}
+        mkdir -p ${{ inputs.docker-images-folder }}
+
+    - if: ${{ inputs.build-docker-images == 'true' }}
+      name: Build cloudbeat binary
+      shell: bash
+      run: |
+        mage -v build
+
+    - if: ${{ inputs.build-docker-images == 'true' }}
+      name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - if: ${{ inputs.build-docker-images == 'true' }}
+      name: Cache docker build cache
+      uses: actions/cache@v4
+      with:
+        path: ${{ inputs.docker-build-cache-folder }}
+        key: ci-buildx-${{ runner.os }}-${{ runner.arch }}-${{ github.workflow }}
+
+    - if: ${{ inputs.build-docker-images == 'true' }}
+      name: Build cloudbeat-docker image
+      uses: docker/build-push-action@v5
+      with:
+        context: .
+        file: ./deploy/Dockerfile
+        push: false
+        tags: cloudbeat:latest
+        cache-from: type=local,src=${{ inputs.docker-build-cache-folder }}/cloudbeat
+        cache-to: type=local,mode=max,dest=${{ inputs.docker-build-cache-folder }}-new/cloudbeat
+        outputs: type=docker,dest=${{ inputs.docker-images-folder }}/cloudbeat.tar
+        no-cache: false
+        platforms: ${{ inputs.docker-build-platforms }}
+
+    - if: ${{ inputs.build-docker-images == 'true' }}
+      name: Build elastic-agent
+      uses: docker/build-push-action@v5
+      env:
+        GOOS: ${{ inputs.goos }}
+        GOARCH: ${{ inputs.goarch }}
+      with:
+        context: .
+        file: ./scripts/packaging/docker/elastic-agent/Dockerfile
+        push: false
+        pull: true
+        tags: ${{ inputs.elastic-agent-docker-image }}:${{ inputs.elastic-agent-docker-image-tag }}
+        cache-from: type=local,src=${{ inputs.docker-build-cache-folder }}/agent
+        cache-to: type=local,mode=max,dest=${{ inputs.docker-build-cache-folder }}-new/agent
+        outputs: type=docker,dest=${{ inputs.docker-images-folder }}/elastic-agent.tar
+        build-args: |
+          ELASTIC_AGENT_IMAGE=docker.elastic.co/beats/elastic-agent:${{ inputs.elastic-agent-docker-image-tag }}
+        no-cache: false
+        platforms: ${{ inputs.docker-build-platforms }}
+
+    - if: ${{ inputs.build-docker-images == 'true' }}
+      name: Build pytest-docker
+      uses: docker/build-push-action@v5
+      with:
+        context: ./tests/.
+        push: false
+        tags: cloudbeat-test:latest
+        cache-from: type=local,src=${{ inputs.docker-build-cache-folder }}/pytest
+        cache-to: type=local,mode=max,dest=${{ inputs.docker-build-cache-folder }}-new/pytest
+        outputs: type=docker,dest=${{ inputs.docker-images-folder }}/pytest.tar
+        no-cache: false
+        platforms: ${{ inputs.docker-build-platforms }}
+
+    - if: ${{ inputs.build-docker-images == 'true' }}
+      name: Rotate cache
+      shell: bash
+      run: |
+        ls -lahR /tmp/ || true
+        [[ -d ${{ inputs.docker-build-cache-folder }}-new ]] && rm -rf ${{ inputs.docker-build-cache-folder }} && mv ${{ inputs.docker-build-cache-folder }}-new ${{ inputs.docker-build-cache-folder }}
+
+    - if: ${{ inputs.build-docker-images == 'true' }}
+      name: Upload docker images
+      uses: actions/upload-artifact@v4
+      with:
+        name: docker-images
+        path: ${{ inputs.docker-images-folder }}
+        if-no-files-found: error
+        retention-days: 2
+        overwrite: true
+
+    - if: ${{ inputs.build-docker-images == 'false' }}
+      name: Download docker images
+      uses: actions/download-artifact@v4
+      with:
+        name: docker-images
+        path: ${{ inputs.docker-images-folder }}