[system testing] Add provision for system testing negative / false-positive scenarios in packages

.buildkite/hooks/pre-command

@@ -33,7 +33,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/private_ci_artifact
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =~ ^integration-parallel ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives) ]]; then
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
 fi
 

.buildkite/pipeline.trigger.integration.tests.sh

@@ -49,6 +49,22 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do
     fi
 done
 
+pushd test/packages/false_positives > /dev/null
+for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
+    package_name=$(basename ${package})
+    echo "      - label: \":go: Running integration test (false positive): ${package_name}\""
+    echo "        key: \"integration-false_positives-${package_name}\""
+    echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}"
+    echo "        env:"
+    echo "          UPLOAD_SAFE_LOGS: 1"
+    echo "        agents:"
+    echo "          provider: \"gcp\""
+    echo "        artifact_paths:"
+    echo "          - build/test-results/*.xml"
+done
+
+ popd > /dev/null
+
 pushd test/packages/parallel > /dev/null
 for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     package_name=$(basename ${package})

.buildkite/scripts/integration_tests.sh

@@ -24,6 +24,7 @@ source .buildkite/scripts/install_deps.sh
 source .buildkite/scripts/tooling.sh
 
 PARALLEL_TARGET="test-check-packages-parallel"
+FALSE_POSITIVES_TARGET="test-check-packages-false-positives"
 KIND_TARGET="test-check-packages-with-kind"
 TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
@@ -108,7 +109,7 @@ if [[ "${TARGET}" == "${KIND_TARGET}" ]]; then
 fi
 
 echo "--- Run integration test ${TARGET}"
-if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then
+if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then
     make install
 
     # allow to fail this command, to be able to upload safe logs

Makefile

@@ -69,14 +69,17 @@ test-stack-command-8x:
 
 test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x
 
-test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks 
+test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives
 
 test-check-packages-with-kind:
 	PACKAGE_TEST_TYPE=with-kind ./scripts/test-check-packages.sh
 
 test-check-packages-other:
 	PACKAGE_TEST_TYPE=other ./scripts/test-check-packages.sh
 
+test-check-packages-false-positives:
+	PACKAGE_TEST_TYPE=false_positives ./scripts/test-check-false-positives.sh
+
 test-check-packages-benchmarks:
 	PACKAGE_TEST_TYPE=benchmarks ./scripts/test-check-packages.sh
 

docs/howto/system_testing.md

@@ -568,6 +568,23 @@ to indexing generated data from the integration's data streams into Elasticsearc
 elastic-package test system --generate
 ```
 
+### System testing negative or false-positive scenarios
+
+The system tests support packages to be tested for negative scenarios. An example would be to test that the `assert.hit_count` is verified when all the docs are ingested rather than just finding enough docs for the testcase.
+
+There are some special rules for testing negative scenarios
+
+- The negative / false-positive test packages are added under `test/packages/false_positives`
+- It is required to have a file `<package_name>.expected_errors` with the lines needed for every package added under `test/packages/false_positives`.
+- One line per error, taking into account that all `\n` were removed, meaning it is just one line for everything.
+- As it is used `grep` with `-E` some kind of regexes can be used.
+
+Example `expected_errors` file content:
+
+```xml
+<testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failure>
+```
+
 ## Continuous Integration
 
 `elastic-package` runs a set of system tests on some [dummy packages](https://github.com/elastic/elastic-package/tree/main/test/packages) to ensure it's functionalities work as expected. This allows to test changes affecting package testing within `elastic-package` before merging and releasing the changes.

internal/testrunner/runners/system/runner.go

@@ -620,6 +620,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
 	// (TODO in future) Optionally exercise service to generate load.
 	logger.Debug("checking for expected data in data stream...")
 	var hits *hits
+	oldHits := 0
 	passed, err := waitUntilTrue(func() (bool, error) {
 		if signal.SIGINT() {
 			return true, errors.New("SIGINT: cancel waiting for policy assigned")
@@ -629,11 +630,21 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
 		hits, err = r.getDocs(dataStream)
 
 		if config.Assert.HitCount > 0 {
-			return hits.size() >= config.Assert.HitCount, err
-		}
+			if hits.size() < config.Assert.HitCount {
+				return false, err
+			}
+
+			ret := hits.size() == oldHits
+			if !ret {
+				oldHits = hits.size()
+				time.Sleep(4 * time.Second)
+			}
 
+			return ret, err
+		}
 		return hits.size() > 0, err
 	}, waitForDataTimeout)
+
 	if err != nil {
 		return result.WithError(err)
 	}

scripts/test-check-false-positives.sh

@@ -0,0 +1,91 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+cleanup() {
+  r=$?
+
+  # Dump stack logs
+  elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-*}}"
+
+  # Take down the stack
+  elastic-package stack down -v
+
+  # Clean used resources
+  for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do
+    (
+      cd $d
+      elastic-package clean -v
+    )
+  done
+
+  # This is a false positive scenario and tests that the test case failure is a success scenario
+  if [ "${PACKAGE_TEST_TYPE:-false_positives}" == "false_positives" ]; then
+    if [ $r == 1 ]; then
+      EXPECTED_ERRORS_FILE="test/packages/false_positives/${PACKAGE_UNDER_TEST}.expected_errors"
+      if [ ! -f ${EXPECTED_ERRORS_FILE} ]; then
+        echo "Error: Missing expected errors file: ${EXPECTED_ERRORS_FILE}"
+      fi
+      RESULTS_NO_SPACES="build/test-results-no-spaces.xml"
+      cat build/test-results/*.xml | tr -d '\n' > ${RESULTS_NO_SPACES}
+
+      # check number of expected errors
+      number_errors=$(cat build/test-results/*.xml | grep "<failure>" | wc -l)
+      expected_errors=$(cat ${EXPECTED_ERRORS_FILE} | wc -l)
+
+      if [ ${number_errors} -ne ${expected_errors} ]; then
+          echo "Error: There are unexpected errors in ${PACKAGE_UNDER_TEST}"
+          exit 1
+      fi
+
+      # check whether or not the expected errors exist in the xml files
+      while read -r line; do
+        cat ${RESULTS_NO_SPACES} | grep -E "${line}"
+      done < ${EXPECTED_ERRORS_FILE}
+      rm -f build/test-results/*.xml
+      rm -f ${RESULTS_NO_SPACES}
+      exit 0
+    elif [ $r == 0 ]; then
+      echo "Error: Expected to fail tests, but there was none failing"
+      exit 1
+    fi
+  fi
+
+  exit $r
+}
+
+trap cleanup EXIT
+
+export ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml"
+
+OLDPWD=$PWD
+# Build/check packages
+for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do
+  (
+    cd $d
+    elastic-package check -v
+  )
+done
+cd -
+
+# Update the stack
+elastic-package stack update -v
+
+# Boot up the stack
+elastic-package stack up -d -v
+
+elastic-package stack status
+
+# Run package tests
+eval "$(elastic-package stack shellinit)"
+
+for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do
+  (
+    cd $d
+    elastic-package install -v
+
+    # defer-cleanup is set to a short period to verify that the option is available
+    elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage
+  )
+cd -
+done

scripts/test-check-packages.sh

@@ -29,7 +29,7 @@ cleanup() {
       elastic-package clean -v
     )
   done
-
+  
   exit $r
 }
 

test/packages/false_positives/httpjson_false_positive_asserts.expected_errors

@@ -0,0 +1 @@
+<testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failure>

test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@v8.8.0

test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/docs/README.md

@@ -0,0 +1,20 @@
+# Custom API input integration
+
+The custom API input integration is used to ingest data from custom RESTful API's that do not currently have an existing integration.
+
+The input itself supports sending both GET and POST requests, transform requests and responses during runtime, paginate and keep a running state on information from the last collected events.
+
+## Configuration
+
+The extensive documentation for the input are currently available
+
+The most commonly used configuration options are available on the main integration page, while more advanced and customizable options currently resides under the "Advanced options" part of the integration settings page.
+
+Configuration is split into three main categories, Request, Response, and Cursor.
+
+The request part of the configuration handles points like which URL endpoint to communicate with, the request body, specific transformations that have to happen before a request is sent out and some custom options like request proxy, timeout and similar options.
+
+The response part of the configuration handles options like transformation, rate limiting, pagination, and splitting the response into different documents before it is sent to Elasticsearch.
+
+The cursor part of the configuration is used when there is a need to keep state between each of the API requests, for example if a timestamp is returned in the response, that should be used as a filter in the next request after that, the cursor is a place where this is stored.
+

test/packages/false_positives/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml

@@ -0,0 +1,14 @@
+version: "2.3"
+services:
+  httpjson:
+    image: docker.elastic.co/observability/stream:v0.7.0
+    ports:
+      - 8080
+    volumes:
+      - ./files:/files:ro
+    environment:
+      PORT: 8080
+    command:
+      - http-server
+      - --addr=:8080
+      - --config=/files/config.yml

test/packages/false_positives/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml

@@ -0,0 +1,21 @@
+rules:
+  - path: /testpagination/api
+    methods: ["GET"]
+    query_params:
+      page: 1
+    request_headers:
+      Authorization: "Basic dGVzdDp0ZXN0"
+    responses:
+      - status_code: 200
+        body: |-
+          {"message": "success", "page": 2}
+  - path: /testpagination/api
+    methods: ["GET"]
+    query_params:
+      page: 2
+    request_headers:
+      Authorization: "Basic dGVzdDp0ZXN0"
+    responses:
+      - status_code: 200
+        body: |-
+          {"message": "success"}

test/packages/false_positives/httpjson_false_positive_asserts/changelog.yml

@@ -0,0 +1,5 @@
+- version: "999.999.999"
+  changes:
+    - description: Test assert.hit_count
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6326

test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml

@@ -0,0 +1,18 @@
+wait_for_data_timeout: 20s
+input: httpjson
+service: httpjson
+data_stream:
+  vars:
+    data_stream.dataset: httpjson.generic
+    username: test
+    password: test
+    request_interval: 5s
+    request_url: http://{{Hostname}}:{{Port}}/testpagination/api?page=1
+    response_pagination: |-
+      - set:
+          target: url.params.page
+          value: '[[.last_response.body.page]]'
+          fail_on_template_error: true
+    enable_request_tracer: true
+assert:
+  hit_count: 2