diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 8dec4bff9f..97ccf5b29b 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -33,7 +33,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/private_ci_artifact # Secrets must be redacted # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables -if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =~ ^integration-parallel ]]; then +if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives) ]]; then export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH}) fi diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh index c232b6616b..744c8a506c 100755 --- a/.buildkite/pipeline.trigger.integration.tests.sh +++ b/.buildkite/pipeline.trigger.integration.tests.sh @@ -49,6 +49,22 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do fi done +pushd test/packages/false_positives > /dev/null +for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do + package_name=$(basename ${package}) + echo " - label: \":go: Running integration test (false positive): ${package_name}\"" + echo " key: \"integration-false_positives-${package_name}\"" + echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}" + echo " env:" + echo " UPLOAD_SAFE_LOGS: 1" + echo " agents:" + echo " provider: \"gcp\"" + echo " artifact_paths:" + echo " - build/test-results/*.xml" +done + + popd > /dev/null + pushd test/packages/parallel > /dev/null for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do package_name=$(basename ${package}) diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh index fb1637c117..1ef5a3d0b1 100755 --- a/.buildkite/scripts/integration_tests.sh +++ b/.buildkite/scripts/integration_tests.sh @@ -24,6 +24,7 @@ source .buildkite/scripts/install_deps.sh source .buildkite/scripts/tooling.sh PARALLEL_TARGET="test-check-packages-parallel" +FALSE_POSITIVES_TARGET="test-check-packages-false-positives" KIND_TARGET="test-check-packages-with-kind" TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX" GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json" @@ -108,7 +109,7 @@ if [[ "${TARGET}" == "${KIND_TARGET}" ]]; then fi echo "--- Run integration test ${TARGET}" -if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then +if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then make install # allow to fail this command, to be able to upload safe logs diff --git a/Makefile b/Makefile index ffa299cc29..d9656e54f4 100644 --- a/Makefile +++ b/Makefile @@ -69,7 +69,7 @@ test-stack-command-8x: test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x -test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks +test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives test-check-packages-with-kind: PACKAGE_TEST_TYPE=with-kind ./scripts/test-check-packages.sh @@ -77,6 +77,9 @@ test-check-packages-with-kind: test-check-packages-other: PACKAGE_TEST_TYPE=other ./scripts/test-check-packages.sh +test-check-packages-false-positives: + PACKAGE_TEST_TYPE=false_positives ./scripts/test-check-false-positives.sh + test-check-packages-benchmarks: PACKAGE_TEST_TYPE=benchmarks ./scripts/test-check-packages.sh diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md index 95f7312a5a..c14a6f4961 100644 --- a/docs/howto/system_testing.md +++ b/docs/howto/system_testing.md @@ -568,6 +568,23 @@ to indexing generated data from the integration's data streams into Elasticsearc elastic-package test system --generate ``` +### System testing negative or false-positive scenarios + +The system tests support packages to be tested for negative scenarios. An example would be to test that the `assert.hit_count` is verified when all the docs are ingested rather than just finding enough docs for the testcase. + +There are some special rules for testing negative scenarios + +- The negative / false-positive test packages are added under `test/packages/false_positives` +- It is required to have a file `.expected_errors` with the lines needed for every package added under `test/packages/false_positives`. +- One line per error, taking into account that all `\n` were removed, meaning it is just one line for everything. +- As it is used `grep` with `-E` some kind of regexes can be used. + +Example `expected_errors` file content: + +```xml + * observed hit count 4 did not match expected hit count 2 +``` + ## Continuous Integration `elastic-package` runs a set of system tests on some [dummy packages](https://github.com/elastic/elastic-package/tree/main/test/packages) to ensure it's functionalities work as expected. This allows to test changes affecting package testing within `elastic-package` before merging and releasing the changes. diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go index b9ca09d4b3..f1af3f0277 100644 --- a/internal/testrunner/runners/system/runner.go +++ b/internal/testrunner/runners/system/runner.go @@ -620,6 +620,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext // (TODO in future) Optionally exercise service to generate load. logger.Debug("checking for expected data in data stream...") var hits *hits + oldHits := 0 passed, err := waitUntilTrue(func() (bool, error) { if signal.SIGINT() { return true, errors.New("SIGINT: cancel waiting for policy assigned") @@ -629,11 +630,21 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext hits, err = r.getDocs(dataStream) if config.Assert.HitCount > 0 { - return hits.size() >= config.Assert.HitCount, err - } + if hits.size() < config.Assert.HitCount { + return false, err + } + + ret := hits.size() == oldHits + if !ret { + oldHits = hits.size() + time.Sleep(4 * time.Second) + } + return ret, err + } return hits.size() > 0, err }, waitForDataTimeout) + if err != nil { return result.WithError(err) } diff --git a/scripts/test-check-false-positives.sh b/scripts/test-check-false-positives.sh new file mode 100755 index 0000000000..d8e82ea5da --- /dev/null +++ b/scripts/test-check-false-positives.sh @@ -0,0 +1,91 @@ +#!/bin/bash + +set -euxo pipefail + +cleanup() { + r=$? + + # Dump stack logs + elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-*}}" + + # Take down the stack + elastic-package stack down -v + + # Clean used resources + for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do + ( + cd $d + elastic-package clean -v + ) + done + + # This is a false positive scenario and tests that the test case failure is a success scenario + if [ "${PACKAGE_TEST_TYPE:-false_positives}" == "false_positives" ]; then + if [ $r == 1 ]; then + EXPECTED_ERRORS_FILE="test/packages/false_positives/${PACKAGE_UNDER_TEST}.expected_errors" + if [ ! -f ${EXPECTED_ERRORS_FILE} ]; then + echo "Error: Missing expected errors file: ${EXPECTED_ERRORS_FILE}" + fi + RESULTS_NO_SPACES="build/test-results-no-spaces.xml" + cat build/test-results/*.xml | tr -d '\n' > ${RESULTS_NO_SPACES} + + # check number of expected errors + number_errors=$(cat build/test-results/*.xml | grep "" | wc -l) + expected_errors=$(cat ${EXPECTED_ERRORS_FILE} | wc -l) + + if [ ${number_errors} -ne ${expected_errors} ]; then + echo "Error: There are unexpected errors in ${PACKAGE_UNDER_TEST}" + exit 1 + fi + + # check whether or not the expected errors exist in the xml files + while read -r line; do + cat ${RESULTS_NO_SPACES} | grep -E "${line}" + done < ${EXPECTED_ERRORS_FILE} + rm -f build/test-results/*.xml + rm -f ${RESULTS_NO_SPACES} + exit 0 + elif [ $r == 0 ]; then + echo "Error: Expected to fail tests, but there was none failing" + exit 1 + fi + fi + + exit $r +} + +trap cleanup EXIT + +export ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml" + +OLDPWD=$PWD +# Build/check packages +for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do + ( + cd $d + elastic-package check -v + ) +done +cd - + +# Update the stack +elastic-package stack update -v + +# Boot up the stack +elastic-package stack up -d -v + +elastic-package stack status + +# Run package tests +eval "$(elastic-package stack shellinit)" + +for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do + ( + cd $d + elastic-package install -v + + # defer-cleanup is set to a short period to verify that the option is available + elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage + ) +cd - +done diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh index 2978bf2136..a1c84e894c 100755 --- a/scripts/test-check-packages.sh +++ b/scripts/test-check-packages.sh @@ -29,7 +29,7 @@ cleanup() { elastic-package clean -v ) done - + exit $r } diff --git a/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors b/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors new file mode 100644 index 0000000000..4cb323f578 --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors @@ -0,0 +1 @@ + * observed hit count 4 did not match expected hit count 2 diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/build.yml b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/build.yml new file mode 100644 index 0000000000..074278e5b1 --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/build.yml @@ -0,0 +1,3 @@ +dependencies: + ecs: + reference: git@v8.8.0 diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/docs/README.md b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/docs/README.md new file mode 100644 index 0000000000..116fd4803a --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/docs/README.md @@ -0,0 +1,20 @@ +# Custom API input integration + +The custom API input integration is used to ingest data from custom RESTful API's that do not currently have an existing integration. + +The input itself supports sending both GET and POST requests, transform requests and responses during runtime, paginate and keep a running state on information from the last collected events. + +## Configuration + +The extensive documentation for the input are currently available + +The most commonly used configuration options are available on the main integration page, while more advanced and customizable options currently resides under the "Advanced options" part of the integration settings page. + +Configuration is split into three main categories, Request, Response, and Cursor. + +The request part of the configuration handles points like which URL endpoint to communicate with, the request body, specific transformations that have to happen before a request is sent out and some custom options like request proxy, timeout and similar options. + +The response part of the configuration handles options like transformation, rate limiting, pagination, and splitting the response into different documents before it is sent to Elasticsearch. + +The cursor part of the configuration is used when there is a need to keep state between each of the API requests, for example if a timestamp is returned in the response, that should be used as a filter in the next request after that, the cursor is a place where this is stored. + diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml new file mode 100644 index 0000000000..9c2cce542f --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml @@ -0,0 +1,14 @@ +version: "2.3" +services: + httpjson: + image: docker.elastic.co/observability/stream:v0.7.0 + ports: + - 8080 + volumes: + - ./files:/files:ro + environment: + PORT: 8080 + command: + - http-server + - --addr=:8080 + - --config=/files/config.yml diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml new file mode 100644 index 0000000000..24902a64c2 --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml @@ -0,0 +1,21 @@ +rules: + - path: /testpagination/api + methods: ["GET"] + query_params: + page: 1 + request_headers: + Authorization: "Basic dGVzdDp0ZXN0" + responses: + - status_code: 200 + body: |- + {"message": "success", "page": 2} + - path: /testpagination/api + methods: ["GET"] + query_params: + page: 2 + request_headers: + Authorization: "Basic dGVzdDp0ZXN0" + responses: + - status_code: 200 + body: |- + {"message": "success"} diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/changelog.yml b/test/packages/false_positives/httpjson_false_positive_asserts/changelog.yml new file mode 100644 index 0000000000..1c605437a2 --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/changelog.yml @@ -0,0 +1,5 @@ +- version: "999.999.999" + changes: + - description: Test assert.hit_count + type: enhancement + link: https://github.com/elastic/integrations/pull/6326 diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml new file mode 100644 index 0000000000..206312f355 --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml @@ -0,0 +1,18 @@ +wait_for_data_timeout: 20s +input: httpjson +service: httpjson +data_stream: + vars: + data_stream.dataset: httpjson.generic + username: test + password: test + request_interval: 5s + request_url: http://{{Hostname}}:{{Port}}/testpagination/api?page=1 + response_pagination: |- + - set: + target: url.params.page + value: '[[.last_response.body.page]]' + fail_on_template_error: true + enable_request_tracer: true +assert: + hit_count: 2 diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/agent/stream/httpjson.yml.hbs b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/agent/stream/httpjson.yml.hbs new file mode 100644 index 0000000000..9ce1088341 --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/agent/stream/httpjson.yml.hbs @@ -0,0 +1,161 @@ +config_version: 2 +data_stream: + dataset: {{data_stream.dataset}} +interval: {{request_interval}} + +{{#if username}} +auth.basic.user: {{username}} +{{/if}} +{{#if password}} +auth.basic.password: {{password}} +{{/if}} + +{{#if pipeline}} +pipeline: {{pipeline}} +{{/if}} + +{{#unless username}} +{{#unless password}} +{{#if oauth_id}} +auth.oauth2.client.id: {{oauth_id}} +{{/if}} +{{#if oauth_secret}} +auth.oauth2.client.secret: {{oauth_secret}} +{{/if}} +{{#if oauth_token_url}} +auth.oauth2.token_url: {{oauth_token_url}} +{{/if}} +{{#if oauth_provider}} +auth.oauth2.provider: {{oauth_provider}} +{{/if}} +{{#if oauth_scopes}} +auth.oauth2.scopes: +{{#each oauth_scopes as |scope i|}} + - {{scope}} +{{/each}} +{{/if}} +{{#if oauth_google_credentials_file}} +auth.oauth2.google.credentials_file: {{oauth_google_credentials_file}} +{{/if}} +{{#if oauth_google_credentials_json}} +auth.oauth2.google.credentials_json: '{{oauth_google_credentials_json}}' +{{/if}} +{{#if oauth_google_jwt_file}} +auth.oauth2.google.jwt_file: {{oauth_google_jwt_file}} +{{/if}} +{{#if oauth_google_jwt_json}} +auth.oauth2.google.jwt_json: {{oauth_google_jwt_json}} +{{/if}} +{{#if oauth_google_delegated_account}} +auth.oauth2.google.delegated_account: {{oauth_google_delegated_account}} +{{/if}} +{{#if oauth_azure_tenant_id}} +auth.oauth2.azure.tenant_id: {{oauth_azure_tenant_id}} +{{/if}} +{{#if oauth_azure_resource}} +auth.oauth2.azure.resource: {{oauth_azure_resource}} +{{/if}} +{{#if oauth_endpoint_params}} +auth.oauth2.endpoint_params: + {{oauth_endpoint_params}} +{{/if}} +{{/unless}} +{{/unless}} + +request.url: {{request_url}} +request.method: {{request_method}} +{{#if request_body}} +request.body: + {{request_body}} +{{/if}} +{{#if request_transforms}} +request.transforms: + {{request_transforms}} +{{/if}} +{{#if request_ssl}} +request.ssl: + {{request_ssl}} +{{/if}} +{{#if request_encode_as}} +request.encode_as: {{request_encode_as}} +{{/if}} +{{#if request_timeout}} +request.timeout: {{request_timeout}} +{{/if}} +{{#if request_proxy_url}} +request.proxy_url: {{request_proxy_url}} +{{/if}} +{{#if request_retry_max_attempts}} +request.retry.max_attempts: {{request_retry_max_attempts}} +{{/if}} +{{#if request_retry_wait_min}} +request.retry.wait_min: {{request_retry_wait_min}} +{{/if}} +{{#if request_retry_wait_max}} +request.retry.wait_max: {{request_retry_wait_max}} +{{/if}} +{{#if request_redirect_forward_headers}} +request.redirect.forward_headers: {{request_redirect_forward_headers}} +{{/if}} +{{#if request_redirect_headers_ban_list}} +request.redirect.headers_ban_list: +{{#each request_redirect_headers_ban_list as |item i|}} + - {{item}} +{{/each}} +{{/if}} +{{#if request_redirect_max_redirects}} +request.redirect.max_redirects: {{request_redirect_max_redirects}} +{{/if}} +{{#if request_rate_limit_limit}} +request.rate_limit.limit: {{request_rate_limit_limit}} +{{/if}} +{{#if request_rate_limit_reset}} +request.rate_limit.reset: {{request_rate_limit_reset}} +{{/if}} +{{#if request_rate_limit_remaining}} +request.rate_limit.remaining: {{request_rate_limit_remaining}} +{{/if}} +{{#if enable_request_tracer}} +request.tracer.filename: "../../logs/httpjson/http-request-trace-*.ndjson" +{{/if}} + +{{#if response_transforms}} +response.transforms: + {{response_transforms}} +{{/if}} +{{#if response_split}} +response.split: + {{response_split}} +{{/if}} +{{#if response_pagination}} +response.pagination: {{response_pagination}} +{{/if}} +{{#if response_decode_as}} +response.decode_as: {{response_decode_as}} +{{/if}} +{{#if response_request_body_on_pagination}} +response.request_body_on_pagination: {{response_request_body_on_pagination}} +{{/if}} + +{{#if cursor}} +cursor: + {{cursor}} +{{/if}} + +{{#if tags}} +tags: +{{#each tags as |tag i|}} + - {{tag}} +{{/each}} +{{/if}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if chain}} +chain: +{{chain}} +{{/if}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/base-fields.yml b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/base-fields.yml new file mode 100644 index 0000000000..d8277624ff --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/base-fields.yml @@ -0,0 +1,20 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: httpjson +- name: event.dataset + type: constant_keyword + description: Event dataset + value: httpjson.generic +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/beats.yml b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/beats.yml new file mode 100644 index 0000000000..ede6958855 --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/beats.yml @@ -0,0 +1,6 @@ +- name: input.type + description: Type of Filebeat input. + type: keyword +- name: tags + type: keyword + description: User defined tags diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/ecs.yml b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/ecs.yml new file mode 100644 index 0000000000..1c3645d5f4 --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/ecs.yml @@ -0,0 +1,6 @@ +- name: ecs.version + external: ecs +- name: event.created + external: ecs +- name: message + external: ecs diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/manifest.yml b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/manifest.yml new file mode 100644 index 0000000000..89b199f86c --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/manifest.yml @@ -0,0 +1,327 @@ +title: Custom API Input +type: logs +streams: + - input: httpjson + description: Collect custom data from REST API's + template_path: httpjson.yml.hbs + title: Custom API Input + vars: + - name: data_stream.dataset + type: text + title: Dataset name + description: | + Dataset to write data to. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html). + default: httpjson.generic + required: true + show_user: true + - name: pipeline + type: text + title: Ingest Pipeline + description: | + The Ingest Node pipeline ID to be used by the integration. + required: false + show_user: true + - name: request_url + type: text + title: Request URL + description: i.e. scheme://host:port/path + show_user: true + required: true + default: https://server.example.com:8089/api + - name: request_interval + type: text + title: Request Interval + description: How often the API is polled, supports seconds, minutes and hours. + show_user: true + required: true + default: 1m + - name: request_method + type: text + title: Request HTTP Method + description: Supports either GET or POST + show_user: true + required: true + default: GET + - name: username + type: text + title: Basic Auth Username + show_user: true + required: false + description: The username to be used with Basic Auth headers + - name: password + type: password + title: Basic Auth Password + show_user: true + required: false + description: The password to be used with Basic Auth headers + - name: oauth_id + type: text + title: Oauth2 Client ID + description: Client ID used for Oauth2 authentication + show_user: true + required: false + - name: oauth_secret + type: password + title: Oauth2 Client Secret + description: Client secret used for Oauth2 authentication + show_user: true + required: false + - name: oauth_token_url + type: text + title: Oauth2 Token URL + description: The URL endpoint that will be used to generate the tokens during the oauth2 flow. It is required if no oauth_custom variable is set or provider is not specified in oauth_custom variable. + show_user: true + required: false + - name: request_body + type: yaml + title: Request Body + description: An optional HTTP body if the request method is POST. All available options can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_body) + show_user: true + multi: false + required: false + - name: request_transforms + type: yaml + title: Request Transforms + description: Optional transformations to perform on the request before it is sent. All available options can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#request-transforms). + show_user: true + multi: false + required: false + - name: response_transforms + type: yaml + title: Response Transforms + description: Optional transformations to perform on the response before it is sent to Elasticsearch. All available options can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#response-transforms). + show_user: true + multi: false + required: false + - name: response_split + type: yaml + title: Response Split + description: Optional transformations to perform on the response to split the response into separate documents before it is sent to Elasticsearch. All available options can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#response-split). + show_user: true + multi: false + required: false + - name: response_pagination + type: yaml + title: Response Pagination + description: Optional settings if pagination is required to retrieve all results. All available options can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#response-pagination). + show_user: true + multi: false + required: false + - name: cursor + type: yaml + title: Custom request cursor + description: | + A cursor is used to keep state between each API request, and can be set to for example the value of something in the response body. + More information can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#cursor). + show_user: true + multi: false + required: false + - name: request_ssl + type: yaml + title: Request SSL Configuration + description: i.e. certificate_authorities, supported_protocols, verification_mode etc, more examples found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-ssl.html#ssl-common-config) + multi: false + required: false + show_user: false + - name: request_encode_as + type: text + title: Request Encode As + description: ContentType used for encoding the request body. If set it will force the encoding in the specified format regardless of the Content-Type header value. + show_user: false + multi: false + required: false + - name: request_timeout + type: text + title: Request Timeout + description: Duration before declaring that the HTTP client connection has timed out. Valid time units are ns, us, ms, s, m, h. Default is "30"s. + show_user: false + multi: false + required: false + - name: request_proxy_url + type: text + title: Request Proxy + description: This specifies proxy configuration in the form of `http[s]://:@:`. + show_user: false + multi: false + required: false + - name: request_retry_max_attempts + type: text + title: Request Retry Max Attempts + description: The maximum number of retries for the HTTP client. Default is "5". + show_user: false + multi: false + required: false + - name: request_retry_wait_min + type: text + title: Request Retry Wait Min + description: The minimum time to wait before a retry is attempted. Default is "1s". + show_user: false + multi: false + required: false + - name: request_retry_wait_max + type: text + title: Request Retry Wait Max + description: The maximum time to wait before a retry is attempted. Default is "60s". + show_user: false + multi: false + required: false + - name: request_redirect_forward_headers + type: bool + title: Request Redirect Forward Headers + description: When set to true request headers are forwarded in case of a redirect. Default is "false". + show_user: false + multi: false + required: false + - name: request_redirect_headers_ban_list + type: text + title: Request Redirect Headers Ban List + description: When Redirect Forward Headers is set to true, all headers except the ones defined in this list will be forwarded. All headers are forwarded by default. + show_user: false + multi: true + required: false + - name: request_redirect_max_redirects + type: text + title: Request Redirect Max Redirects + description: The maximum number of redirects to follow for a request. Default is "10". + show_user: false + multi: false + required: false + - name: request_rate_limit_limit + type: text + title: Request Rate Limit + description: The value of the response that specifies the total limit. It is defined with a Go template value. + show_user: false + multi: false + required: false + - name: request_rate_limit_reset + type: text + title: Request Rate Limit Reset + description: The value of the response that specifies the epoch time when the rate limit will reset. It is defined with a Go template value. + show_user: false + multi: false + required: false + - name: request_rate_limit_remaining + type: text + title: Request Rate Limit Remaining + description: The value of the response that specifies the remaining quota of the rate limit. It is defined with a Go template value. + show_user: false + multi: false + required: false + - name: oauth_provider + type: text + title: Oauth2 Provider + description: Used to configure supported oauth2 providers. Each supported provider will require specific settings. It is not set by default. Supported providers are "azure" and "google". + show_user: false + multi: false + required: false + - name: oauth_scopes + type: text + title: Oauth2 Scopes + description: A list of scopes that will be requested during the oauth2 flow. It is optional for all providers. + show_user: false + multi: true + required: false + - name: oauth_google_credentials_file + type: text + title: Oauth2 Google Credentials File + description: The full path to the credentials file for Google. + show_user: false + multi: false + required: false + - name: oauth_google_credentials_json + type: text + title: Oauth2 Google Credentials JSON + description: Your Google credentials information as raw JSON. + show_user: false + multi: false + required: false + - name: oauth_google_jwt_file + type: text + title: Oauth2 Google JWT File + description: Full path to the JWT Account Key file for Google. + show_user: false + multi: false + required: false + - name: oauth_google_jwt_json + type: text + title: Oauth2 Google JWT JSON + description: Your Google JWT information as raw JSON. + multi: false + required: false + show_user: false + - name: oauth_google_delegated_account + type: text + title: Oauth2 Google Delegated account + description: Email of the delegated account used to create the credentials (usually an admin). + show_user: false + multi: false + required: false + - name: oauth_azure_tenant_id + type: text + title: Oauth2 Azure Tenant ID + description: Optional setting used for authentication when using Azure provider. Since it is used in the process to generate the token_url, it can’t be used in combination with it. + show_user: false + multi: false + required: false + - name: oauth_azure_resource + type: text + title: Oauth2 Azure Resource + description: Optional setting for the accessed WebAPI resource when using azure provider. + show_user: false + multi: false + required: false + - name: oauth_endpoint_params + type: yaml + title: Oauth2 Endpoint Params + description: Set of values that will be sent on each request to the token_url. Each param key can have multiple values. Can be set for all providers except google. + show_user: false + multi: false + required: false + - name: response_decode_as + type: text + title: Response decode settings + description: | + ContentType used for decoding the response body. Supported values: application/json, application/x-ndjson. By default it will use what is in the response Content-Type header. + show_user: false + required: false + - name: response_request_body_on_pagination + type: bool + title: Include request body on Pagination + description: | + If set to true, the values in request.body are sent with pagination requests. + show_user: false + multi: false + required: false + - name: chain + type: yaml + title: Chain + multi: false + required: false + show_user: false + description: | + A list of requests to be made after the first one. See [`chain`](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#chain) for details. + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: > + Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. + + - name: enable_request_tracer + type: bool + title: Enable request tracing + multi: false + required: false + show_user: false + description: > + The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details. + + - name: tags + type: text + title: Tags + multi: true + show_user: false + default: + - forwarded diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/sample_event.json b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/sample_event.json new file mode 100644 index 0000000000..97f5b56929 --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/sample_event.json @@ -0,0 +1,36 @@ +{ + "@timestamp": "2022-03-10T12:47:55.098Z", + "agent": { + "ephemeral_id": "03c96875-43cc-4abc-b998-99527ff31de3", + "id": "0ddbfef9-4d38-400d-8404-d2df456bddc0", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.0.0" + }, + "data_stream": { + "dataset": "httpjson.generic", + "namespace": "ep", + "type": "logs" + }, + "ecs": { + "version": "8.2.0" + }, + "elastic_agent": { + "id": "0ddbfef9-4d38-400d-8404-d2df456bddc0", + "snapshot": false, + "version": "8.0.0" + }, + "event": { + "agent_id_status": "verified", + "created": "2022-03-10T12:47:55.098Z", + "dataset": "httpjson.generic", + "ingested": "2022-03-10T12:47:56Z" + }, + "input": { + "type": "httpjson" + }, + "message": "{\"message\":\"success\",\"page\":2}", + "tags": [ + "forwarded" + ] +} \ No newline at end of file diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/docs/README.md b/test/packages/false_positives/httpjson_false_positive_asserts/docs/README.md new file mode 100644 index 0000000000..116fd4803a --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/docs/README.md @@ -0,0 +1,20 @@ +# Custom API input integration + +The custom API input integration is used to ingest data from custom RESTful API's that do not currently have an existing integration. + +The input itself supports sending both GET and POST requests, transform requests and responses during runtime, paginate and keep a running state on information from the last collected events. + +## Configuration + +The extensive documentation for the input are currently available + +The most commonly used configuration options are available on the main integration page, while more advanced and customizable options currently resides under the "Advanced options" part of the integration settings page. + +Configuration is split into three main categories, Request, Response, and Cursor. + +The request part of the configuration handles points like which URL endpoint to communicate with, the request body, specific transformations that have to happen before a request is sent out and some custom options like request proxy, timeout and similar options. + +The response part of the configuration handles options like transformation, rate limiting, pagination, and splitting the response into different documents before it is sent to Elasticsearch. + +The cursor part of the configuration is used when there is a need to keep state between each of the API requests, for example if a timestamp is returned in the response, that should be used as a filter in the next request after that, the cursor is a place where this is stored. + diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/manifest.yml b/test/packages/false_positives/httpjson_false_positive_asserts/manifest.yml new file mode 100644 index 0000000000..4da9f431ef --- /dev/null +++ b/test/packages/false_positives/httpjson_false_positive_asserts/manifest.yml @@ -0,0 +1,20 @@ +format_version: 2.7.0 +name: httpjson +title: Custom API +description: Collect custom events from an API endpoint with Elastic agent +type: integration +version: "999.999.999" +conditions: + kibana.version: "^8.7.1" +categories: + - custom +policy_templates: + - name: generic + title: Custom API Input + description: Collect custom data from REST API's + inputs: + - type: httpjson + title: Collect custom data from REST API's + description: Collect custom data from REST API's +owner: + github: elastic/security-external-integrations