Merge pull request #290 from cavokz/update-rules #2027

Workflow file for this run

	name: Main

	on:
	push:
	tags: "v[0-9]+.[0-9]+.[0-9]+"
	branches: "main"
	pull_request:
	branches: "*"
	schedule:
	# every Monday at 3:30 AM
	- cron: "30 3 * * 1"

	env:
	TEST_VERBOSITY: 2

	jobs:
	license-check:
	name: License check
	runs-on: ubuntu-latest

	steps:
	- name: Checkout code
	uses: actions/checkout@v3

	- name: License check
	run: make license-check

	lint-check:
	name: Code lint check
	runs-on: ubuntu-latest

	steps:
	- name: Checkout code
	uses: actions/checkout@v3

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.12"

	- name: Setup Go
	uses: actions/setup-go@v4
	with:
	go-version: "1.21"
	cache: false

	- name: Install dependencies
	run: make prereq-lint

	- name: Lint
	run: make lint cli-lint

	unit-tests:
	name: Unit tests (${{ matrix.os }}/py-${{ matrix.python-version }})
	runs-on: ${{ matrix.os }}

	strategy:
	fail-fast: false
	matrix:
	stack-version: ["8.2.0"]
	schema-uri: ["./etc/ecs-v8.2.1.tar.gz"]
	python-version: ["3.8", "3.12"]
	os: ["ubuntu-latest", "macos-latest"]

	steps:
	- name: Checkout code
	uses: actions/checkout@v3

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install dependencies
	run: make prereq

	- name: Run unit tests
	env:
	TEST_STACK_VERSION: ${{ matrix.stack-version }}
	TEST_SCHEMA_URI: ${{ matrix.schema-uri }}
	TEST_DETECTION_RULES_URI: "https://epr.elastic.co/search?package=security_detection_engine&kibana.version=${{ matrix.stack-version }}"
	run: make tests

	online-tests:
	name: Online tests (${{ matrix.stack-version }})
	runs-on: "ubuntu-latest"
	needs:
	- unit-tests

	strategy:
	fail-fast: false
	matrix:
	include:
	- stack-version: 8.13.0
	schema-uri: "./etc/ecs-v8.11.0.tar.gz"
	- stack-version: 8.12.0
	schema-uri: "./etc/ecs-v8.11.0.tar.gz"
	- stack-version: 8.11.0
	schema-uri: "./etc/ecs-v8.11.0.tar.gz"
	- stack-version: 8.10.1
	schema-uri: "./etc/ecs-v8.10.0.tar.gz"
	- stack-version: 8.9.0
	schema-uri: "./etc/ecs-v8.9.0.tar.gz"
	- stack-version: 8.8.0
	schema-uri: "./etc/ecs-v8.8.0.tar.gz"
	- stack-version: 8.7.0
	schema-uri: "./etc/ecs-v8.7.0.tar.gz"
	- stack-version: 8.6.0
	schema-uri: "./etc/ecs-v8.6.1.tar.gz"
	- stack-version: 8.5.0
	schema-uri: "./etc/ecs-v8.5.2.tar.gz"
	- stack-version: 8.4.0
	schema-uri: "./etc/ecs-v8.4.0.tar.gz"
	- stack-version: 8.3.0
	schema-uri: "./etc/ecs-v8.3.1.tar.gz"
	- stack-version: 8.2.0
	schema-uri: "./etc/ecs-v8.2.1.tar.gz"

	env:
	TEST_ELASTICSEARCH_URL: "http://elastic:changeme@localhost:29650"
	TEST_KIBANA_URL: "http://elastic:changeme@localhost:65290"
	TEST_GENEVE_LOG: "gnv-${{ matrix.stack-version }}.log"

	steps:
	- name: Checkout code
	uses: actions/checkout@v3

	- name: Setup Go
	uses: actions/setup-go@v4
	with:
	go-version: "1.18"
	cache: false

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.8"

	- name: Install dependencies
	run: make prereq

	- name: Build CLI
	run: make pygolo-diags cli-build

	- name: Start Geneve server
	run: \|
	./gnv serve -l 0.0.0.0:9280 -r $TEST_ELASTICSEARCH_URL --log $TEST_GENEVE_LOG &
	curl -s --retry 3 --retry-connrefused http://localhost:9256/api/status
	curl -s -XPOST -H "Content-Type: application/yaml" http://localhost:9256/api/grasp/ignore --data-binary '@tests/grasp-ignore.yaml'

	- name: Setup cache
	id: elastic-stack-cache
	uses: actions/cache@v3
	with:
	key: elastic-stack-cache-${{ matrix.stack-version }}
	path: ~/elastic-stack-cache

	- name: Reuse Elastic Stack ${{ matrix.stack-version }} Docker images
	if: steps.elastic-stack-cache.outputs.cache-hit == 'true'
	run: \|
	docker load -i ~/elastic-stack-cache/elasticsearch-${{ matrix.stack-version }}.tar
	docker load -i ~/elastic-stack-cache/kibana-${{ matrix.stack-version }}.tar

	- name: Pull Elastic Stack ${{ matrix.stack-version }} Docker images
	if: steps.elastic-stack-cache.outputs.cache-hit != 'true'
	env:
	TEST_STACK_VERSION: ${{ matrix.stack-version }}
	run: \|
	mkdir ~/elastic-stack-cache
	docker compose pull -q
	docker save -o ~/elastic-stack-cache/elasticsearch-${{ matrix.stack-version }}.tar \
	docker.elastic.co/elasticsearch/elasticsearch:${{ matrix.stack-version }}
	docker save -o ~/elastic-stack-cache/kibana-${{ matrix.stack-version }}.tar \
	docker.elastic.co/kibana/kibana:${{ matrix.stack-version }}

	- name: Start Elastic Stack ${{ matrix.stack-version }}
	env:
	TEST_STACK_VERSION: ${{ matrix.stack-version }}
	TEST_ELASTICSEARCH_PROXY: "http://host.docker.internal:9280"
	run: make up

	- name: Run online tests
	env:
	TEST_STACK_VERSION: ${{ matrix.stack-version }}
	TEST_SCHEMA_URI: ${{ matrix.schema-uri }}
	TEST_DETECTION_RULES_URI: "https://epr.elastic.co/search?package=security_detection_engine&kibana.version=${{ matrix.stack-version }}"
	TEST_SIGNALS_QUERIES: 1
	TEST_SIGNALS_RULES: 1
	TEST_ELASTICSEARCH_URL: "http://localhost:9280"
	run: make online-tests

	- name: Explain shards allocation
	if: always()
	run: curl -s $TEST_ELASTICSEARCH_URL/_cat/shards?v

	- name: Print Stack logs
	if: always()
	run: docker compose logs

	- name: Stop Elastic Stack ${{ matrix.stack-version }}
	if: always()
	run: \|
	docker logs geneve-test-es-1 >es-${{ matrix.stack-version }}.log
	docker logs geneve-test-kbn-1 >kbn-${{ matrix.stack-version }}.log
	make down

	- name: Stop Geneve server
	if: always()
	run: \|
	curl -s "http://localhost:9256/api/grasp"
	curl -s "http://localhost:9256/api/grasp/indices?percent=100"
	curl -s "http://localhost:9256/api/grasp/calls?percent=100"
	curl -s "http://localhost:9256/api/grasp/searches?percent=100"
	killall gnv
	if grep "WARNING: DATA RACE" $TEST_GENEVE_LOG; then awk '/WARNING: DATA RACE/{found=1} found' $TEST_GENEVE_LOG; false; fi

	- name: Upload logs
	if: always()
	uses: actions/upload-artifact@v3
	with:
	name: logs-${{ matrix.stack-version }}
	path: \|
	es-*.log
	kbn-*.log
	gnv-*.log

	- name: Upload reports
	if: always()
	uses: actions/upload-artifact@v3
	with:
	name: test-reports-${{ matrix.stack-version }}
	path: tests/reports/*.new.md

	cli:
	name: CLI (${{ matrix.os }}/go-${{ matrix.go-version }}/py-${{ matrix.python-version }})
	runs-on: ${{ matrix.os }}

	strategy:
	fail-fast: false
	matrix:
	go-version: ["1.18", "1.21"]
	python-version: ["3.8", "3.12"]
	os: ["ubuntu-latest", "macos-latest"]

	steps:
	- name: Checkout code
	uses: actions/checkout@v3

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Setup Go
	uses: actions/setup-go@v4
	with:
	go-version: ${{ matrix.go-version }}
	cache: false

	- name: Install dependencies
	run: make prereq

	- name: Build
	run: make pygolo-diags cli-build

	- name: Test
	run: make cli-test

	- name: Benchmark
	run: make cli-bench

	package-build:
	name: Package build (${{ matrix.os }}/py-${{ matrix.python-version }})
	runs-on: ${{ matrix.os }}

	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.8", "3.12"]
	os: ["ubuntu-latest", "macos-latest"]

	steps:
	- name: Checkout code
	uses: actions/checkout@v3

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Install dependencies
	run: make prereq

	- name: Build package
	run: make package

	- name: Upload artifacts
	uses: actions/upload-artifact@v3
	with:
	name: python-package-${{ matrix.python-version }}-${{ matrix.os }}
	path: dist/*

	package-tests:
	name: Package tests (${{ matrix.os }}/py-${{ matrix.python-version }})
	runs-on: ${{ matrix.os }}
	needs: package-build

	strategy:
	fail-fast: false
	matrix:
	python-version: ["3.8", "3.12"]
	os: ["ubuntu-latest", "macos-latest"]

	steps:
	- name: Checkout code
	uses: actions/checkout@v3
	with:
	path: code

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Download artifacts
	uses: actions/download-artifact@v3
	with:
	name: python-package-${{ matrix.python-version }}-${{ matrix.os }}
	path: dist

	- name: Install package
	run: make -f code/Makefile pkg-install

	publish:
	name: Publish
	runs-on: ubuntu-latest
	needs:
	- license-check
	- lint-check
	- unit-tests
	- online-tests
	- package-tests
	- cli
	if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')

	steps:
	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.8"

	- name: Download artifacts
	uses: actions/download-artifact@v3
	with:
	name: python-package-3.8-ubuntu-latest
	path: dist

	- name: Publish to TestPyPI
	uses: pypa/gh-action-pypi-publish@v1.5.0
	if: github.repository != 'elastic/geneve'
	with:
	password: ${{ secrets.TEST_PYPI_API_TOKEN }}
	print_hash: true
	repository_url: https://test.pypi.org/legacy/
	skip_existing: true

	- name: Publish to PyPI
	uses: pypa/gh-action-pypi-publish@v1.5.0
	if: github.repository == 'elastic/geneve'
	with:
	password: ${{ secrets.PYPI_API_TOKEN }}
	print_hash: true

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Merge pull request #290 from cavokz/update-rules #2027

Workflow file

Merge pull request #290 from cavokz/update-rules #2027

Jobs

Run details

Workflow file for this run