Main #2041
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Main | |
| on: | |
| push: | |
| tags: "v[0-9]+.[0-9]+.[0-9]+" | |
| branches: "main" | |
| pull_request: | |
| branches: "*" | |
| schedule: | |
| # every Monday at 3:30 AM | |
| - cron: "30 3 * * 1" | |
| env: | |
| TEST_VERBOSITY: 2 | |
| jobs: | |
| license-check: | |
| name: License check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: License check | |
| run: make license-check | |
| lint-check: | |
| name: Code lint check | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.12" | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.22" | |
| cache: false | |
| - name: Install dependencies | |
| run: make prereq-lint | |
| - name: Lint | |
| run: make lint cli-lint | |
| unit-tests: | |
| name: Unit tests (${{ matrix.os }}/py-${{ matrix.python-version }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| stack-version: ["8.2.0"] | |
| schema-uri: ["./etc/ecs-v8.2.1.tar.gz"] | |
| python-version: ["3.8", "3.12"] | |
| os: ["ubuntu-latest", "macos-latest"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: make prereq | |
| - name: Run unit tests | |
| env: | |
| TEST_STACK_VERSION: ${{ matrix.stack-version }} | |
| TEST_SCHEMA_URI: ${{ matrix.schema-uri }} | |
| TEST_DETECTION_RULES_URI: "https://epr.elastic.co/search?package=security_detection_engine&kibana.version=${{ matrix.stack-version }}" | |
| run: make tests | |
| online-tests: | |
| name: Online tests (${{ matrix.stack-version }}) | |
| runs-on: "ubuntu-latest" | |
| needs: | |
| - unit-tests | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - stack-version: 8.14.0 | |
| schema-uri: "./etc/ecs-v8.11.0.tar.gz" | |
| - stack-version: 8.13.0 | |
| schema-uri: "./etc/ecs-v8.11.0.tar.gz" | |
| - stack-version: 8.12.0 | |
| schema-uri: "./etc/ecs-v8.11.0.tar.gz" | |
| - stack-version: 8.11.0 | |
| schema-uri: "./etc/ecs-v8.11.0.tar.gz" | |
| - stack-version: 8.10.1 | |
| schema-uri: "./etc/ecs-v8.10.0.tar.gz" | |
| - stack-version: 8.9.0 | |
| schema-uri: "./etc/ecs-v8.9.0.tar.gz" | |
| - stack-version: 8.8.0 | |
| schema-uri: "./etc/ecs-v8.8.0.tar.gz" | |
| - stack-version: 8.7.0 | |
| schema-uri: "./etc/ecs-v8.7.0.tar.gz" | |
| - stack-version: 8.6.0 | |
| schema-uri: "./etc/ecs-v8.6.1.tar.gz" | |
| - stack-version: 8.5.0 | |
| schema-uri: "./etc/ecs-v8.5.2.tar.gz" | |
| - stack-version: 8.4.0 | |
| schema-uri: "./etc/ecs-v8.4.0.tar.gz" | |
| - stack-version: 8.3.0 | |
| schema-uri: "./etc/ecs-v8.3.1.tar.gz" | |
| - stack-version: 8.2.0 | |
| schema-uri: "./etc/ecs-v8.2.1.tar.gz" | |
| env: | |
| TEST_ELASTICSEARCH_URL: "http://elastic:changeme@localhost:29650" | |
| TEST_KIBANA_URL: "http://elastic:changeme@localhost:65290" | |
| TEST_GENEVE_LOG: "gnv-${{ matrix.stack-version }}.log" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.18" | |
| cache: false | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.8" | |
| - name: Install dependencies | |
| run: make prereq | |
| - name: Build CLI | |
| run: make pygolo-diags cli-build | |
| - name: Start Geneve server | |
| run: | | |
| ./gnv serve -l 0.0.0.0:9280 -r $TEST_ELASTICSEARCH_URL --log $TEST_GENEVE_LOG & | |
| curl -s --retry 3 --retry-connrefused http://localhost:9256/api/status | |
| curl -s -XPOST -H "Content-Type: application/yaml" http://localhost:9256/api/grasp/ignore --data-binary '@tests/grasp-ignore.yaml' | |
| - name: Setup cache | |
| id: elastic-stack-cache | |
| uses: actions/cache@v3 | |
| with: | |
| key: elastic-stack-cache-${{ matrix.stack-version }} | |
| path: ~/elastic-stack-cache | |
| - name: Reuse Elastic Stack ${{ matrix.stack-version }} Docker images | |
| if: steps.elastic-stack-cache.outputs.cache-hit == 'true' | |
| run: | | |
| docker load -i ~/elastic-stack-cache/elasticsearch-${{ matrix.stack-version }}.tar | |
| docker load -i ~/elastic-stack-cache/kibana-${{ matrix.stack-version }}.tar | |
| - name: Pull Elastic Stack ${{ matrix.stack-version }} Docker images | |
| if: steps.elastic-stack-cache.outputs.cache-hit != 'true' | |
| env: | |
| TEST_STACK_VERSION: ${{ matrix.stack-version }} | |
| run: | | |
| mkdir ~/elastic-stack-cache | |
| docker compose pull -q | |
| docker save -o ~/elastic-stack-cache/elasticsearch-${{ matrix.stack-version }}.tar \ | |
| docker.elastic.co/elasticsearch/elasticsearch:${{ matrix.stack-version }} | |
| docker save -o ~/elastic-stack-cache/kibana-${{ matrix.stack-version }}.tar \ | |
| docker.elastic.co/kibana/kibana:${{ matrix.stack-version }} | |
| - name: Start Elastic Stack ${{ matrix.stack-version }} | |
| env: | |
| TEST_STACK_VERSION: ${{ matrix.stack-version }} | |
| TEST_ELASTICSEARCH_PROXY: "http://host.docker.internal:9280" | |
| run: make up | |
| - name: Run online tests | |
| env: | |
| TEST_STACK_VERSION: ${{ matrix.stack-version }} | |
| TEST_SCHEMA_URI: ${{ matrix.schema-uri }} | |
| TEST_DETECTION_RULES_URI: "https://epr.elastic.co/search?package=security_detection_engine&kibana.version=${{ matrix.stack-version }}" | |
| TEST_SIGNALS_QUERIES: 1 | |
| TEST_SIGNALS_RULES: 1 | |
| TEST_ELASTICSEARCH_URL: "http://localhost:9280" | |
| run: make online-tests | |
| - name: Explain shards allocation | |
| if: always() | |
| run: curl -s $TEST_ELASTICSEARCH_URL/_cat/shards?v | |
| - name: Print Stack logs | |
| if: always() | |
| run: docker compose logs | |
| - name: Stop Elastic Stack ${{ matrix.stack-version }} | |
| if: always() | |
| run: | | |
| docker logs geneve-test-es-1 >es-${{ matrix.stack-version }}.log | |
| docker logs geneve-test-kbn-1 >kbn-${{ matrix.stack-version }}.log | |
| make down | |
| - name: Stop Geneve server | |
| if: always() | |
| run: | | |
| curl -s "http://localhost:9256/api/grasp" | |
| curl -s "http://localhost:9256/api/grasp/indices?percent=100" | |
| curl -s "http://localhost:9256/api/grasp/calls?percent=100" | |
| curl -s "http://localhost:9256/api/grasp/searches?percent=100" | |
| killall gnv | |
| if grep "WARNING: DATA RACE" $TEST_GENEVE_LOG; then awk '/WARNING: DATA RACE/{found=1} found' $TEST_GENEVE_LOG; false; fi | |
| - name: Upload logs | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: logs-${{ matrix.stack-version }} | |
| path: | | |
| es-*.log | |
| kbn-*.log | |
| gnv-*.log | |
| - name: Upload reports | |
| if: always() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: test-reports-${{ matrix.stack-version }} | |
| path: tests/reports/*.new.md | |
| cli: | |
| name: CLI (${{ matrix.os }}/go-${{ matrix.go-version }}/py-${{ matrix.python-version }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| go-version: ["1.18", "1.22"] | |
| python-version: ["3.8", "3.12"] | |
| os: ["ubuntu-latest", "macos-latest"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ matrix.go-version }} | |
| cache: false | |
| - name: Install dependencies | |
| run: make prereq | |
| - name: Build | |
| run: make pygolo-diags cli-build | |
| - name: Test | |
| run: make cli-test | |
| - name: Benchmark | |
| run: make cli-bench | |
| package-build: | |
| name: Package build (${{ matrix.os }}/py-${{ matrix.python-version }}) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.12"] | |
| os: ["ubuntu-latest", "macos-latest"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: make prereq | |
| - name: Build package | |
| run: make package | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: python-package-${{ matrix.python-version }}-${{ matrix.os }} | |
| path: dist/* | |
| package-tests: | |
| name: Package tests (${{ matrix.os }}/py-${{ matrix.python-version }}) | |
| runs-on: ${{ matrix.os }} | |
| needs: package-build | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.8", "3.12"] | |
| os: ["ubuntu-latest", "macos-latest"] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| path: code | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-package-${{ matrix.python-version }}-${{ matrix.os }} | |
| path: dist | |
| - name: Install package | |
| run: make -f code/Makefile pkg-install | |
| publish: | |
| name: Publish | |
| runs-on: ubuntu-latest | |
| needs: | |
| - license-check | |
| - lint-check | |
| - unit-tests | |
| - online-tests | |
| - package-tests | |
| - cli | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') | |
| steps: | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.8" | |
| - name: Download artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: python-package-3.8-ubuntu-latest | |
| path: dist | |
| - name: Publish to TestPyPI | |
| uses: pypa/gh-action-pypi-publish@v1.5.0 | |
| if: github.repository != 'elastic/geneve' | |
| with: | |
| password: ${{ secrets.TEST_PYPI_API_TOKEN }} | |
| print_hash: true | |
| repository_url: https://test.pypi.org/legacy/ | |
| skip_existing: true | |
| - name: Publish to PyPI | |
| uses: pypa/gh-action-pypi-publish@v1.5.0 | |
| if: github.repository == 'elastic/geneve' | |
| with: | |
| password: ${{ secrets.PYPI_API_TOKEN }} | |
| print_hash: true |