From 10b905a8e1ae922a968a55019335a703e2aea224 Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Thu, 29 Jun 2023 12:09:26 +0200
Subject: [PATCH 01/19] Recheck for more hits

---
 internal/testrunner/runners/system/runner.go | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index b9ca09d4b3..1227331938 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -620,6 +620,7 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
 	// (TODO in future) Optionally exercise service to generate load.
 	logger.Debug("checking for expected data in data stream...")
 	var hits *hits
+	oldHits := 0
 	passed, err := waitUntilTrue(func() (bool, error) {
 		if signal.SIGINT() {
 			return true, errors.New("SIGINT: cancel waiting for policy assigned")
@@ -627,13 +628,18 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
 
 		var err error
 		hits, err = r.getDocs(dataStream)
+		if hits.size() == 0 {
+			return false, err
+		}
 
-		if config.Assert.HitCount > 0 {
-			return hits.size() >= config.Assert.HitCount, err
+		ret := hits.size() == oldHits
+		if !ret {
+			oldHits = hits.size()
 		}
 
-		return hits.size() > 0, err
+		return ret, err
 	}, waitForDataTimeout)
+
 	if err != nil {
 		return result.WithError(err)
 	}

From 9bc8c61a7be446e6469fe0c650200b469a3754cb Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Thu, 29 Jun 2023 12:27:16 +0200
Subject: [PATCH 02/19] Change the retry interval to 5 seconds

---
 internal/testrunner/runners/system/runner.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index 1227331938..a8815754cd 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -987,7 +987,7 @@ func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error)
 	timeoutTicker := time.NewTicker(timeout)
 	defer timeoutTicker.Stop()
 
-	retryTicker := time.NewTicker(1 * time.Second)
+	retryTicker := time.NewTicker(5 * time.Second)
 	defer retryTicker.Stop()
 
 	for {

From bf232cc7cfbfce422bc99f9e2e6e3d7a9584cc02 Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Fri, 30 Jun 2023 10:56:38 +0200
Subject: [PATCH 03/19] Add a test for false positives

---
 scripts/test-check-packages.sh                |   9 +
 .../_dev/build/build.yml                      |   3 +
 .../_dev/build/docs/README.md                 |  20 ++
 .../_dev/deploy/docker/docker-compose.yml     |  14 +
 .../_dev/deploy/docker/files/config.yml       |  21 ++
 .../changelog.yml                             |   5 +
 .../test/system/test-pagination-config.yml    |  18 +
 .../generic/agent/stream/httpjson.yml.hbs     | 161 +++++++++
 .../generic/fields/base-fields.yml            |  20 ++
 .../data_stream/generic/fields/beats.yml      |   6 +
 .../data_stream/generic/fields/ecs.yml        |   6 +
 .../data_stream/generic/manifest.yml          | 327 ++++++++++++++++++
 .../data_stream/generic/sample_event.json     |  36 ++
 .../docs/README.md                            |  20 ++
 .../manifest.yml                              |  20 ++
 15 files changed, 686 insertions(+)
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/_dev/build/build.yml
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/_dev/build/docs/README.md
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/changelog.yml
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/data_stream/generic/agent/stream/httpjson.yml.hbs
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/base-fields.yml
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/beats.yml
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/ecs.yml
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/data_stream/generic/manifest.yml
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/data_stream/generic/sample_event.json
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/docs/README.md
 create mode 100644 test/packages/other/httpjson_false_positive_asserts/manifest.yml

diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh
index 2978bf2136..5397e5a978 100755
--- a/scripts/test-check-packages.sh
+++ b/scripts/test-check-packages.sh
@@ -30,6 +30,15 @@ cleanup() {
     )
   done
 
+  # This is a false positive scenario and tests that the test case failure is a success scenario  
+  if [ "${PACKAGE_TEST_TYPE:-other}" == "other" ] && [ "${PACKAGE_UNDER_TEST:-*}" == "httpjson_false_positive_asserts" ]; then
+    if [ $r == 1 ]; then
+        exit 0
+      elif [ $r == 0 ]; then
+        exit 1
+      fi
+  fi
+
   exit $r
 }
 
diff --git a/test/packages/other/httpjson_false_positive_asserts/_dev/build/build.yml b/test/packages/other/httpjson_false_positive_asserts/_dev/build/build.yml
new file mode 100644
index 0000000000..875463aaf4
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/_dev/build/build.yml
@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@8.8
diff --git a/test/packages/other/httpjson_false_positive_asserts/_dev/build/docs/README.md b/test/packages/other/httpjson_false_positive_asserts/_dev/build/docs/README.md
new file mode 100644
index 0000000000..116fd4803a
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/_dev/build/docs/README.md
@@ -0,0 +1,20 @@
+# Custom API input integration
+
+The custom API input integration is used to ingest data from custom RESTful API's that do not currently have an existing integration.
+
+The input itself supports sending both GET and POST requests, transform requests and responses during runtime, paginate and keep a running state on information from the last collected events.
+
+## Configuration
+
+The extensive documentation for the input are currently available
+
+The most commonly used configuration options are available on the main integration page, while more advanced and customizable options currently resides under the "Advanced options" part of the integration settings page.
+
+Configuration is split into three main categories, Request, Response, and Cursor.
+
+The request part of the configuration handles points like which URL endpoint to communicate with, the request body, specific transformations that have to happen before a request is sent out and some custom options like request proxy, timeout and similar options.
+
+The response part of the configuration handles options like transformation, rate limiting, pagination, and splitting the response into different documents before it is sent to Elasticsearch.
+
+The cursor part of the configuration is used when there is a need to keep state between each of the API requests, for example if a timestamp is returned in the response, that should be used as a filter in the next request after that, the cursor is a place where this is stored.
+
diff --git a/test/packages/other/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml b/test/packages/other/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml
new file mode 100644
index 0000000000..9c2cce542f
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml
@@ -0,0 +1,14 @@
+version: "2.3"
+services:
+  httpjson:
+    image: docker.elastic.co/observability/stream:v0.7.0
+    ports:
+      - 8080
+    volumes:
+      - ./files:/files:ro
+    environment:
+      PORT: 8080
+    command:
+      - http-server
+      - --addr=:8080
+      - --config=/files/config.yml
diff --git a/test/packages/other/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml b/test/packages/other/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml
new file mode 100644
index 0000000000..24902a64c2
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml
@@ -0,0 +1,21 @@
+rules:
+  - path: /testpagination/api
+    methods: ["GET"]
+    query_params:
+      page: 1
+    request_headers:
+      Authorization: "Basic dGVzdDp0ZXN0"
+    responses:
+      - status_code: 200
+        body: |-
+          {"message": "success", "page": 2}
+  - path: /testpagination/api
+    methods: ["GET"]
+    query_params:
+      page: 2
+    request_headers:
+      Authorization: "Basic dGVzdDp0ZXN0"
+    responses:
+      - status_code: 200
+        body: |-
+          {"message": "success"}
diff --git a/test/packages/other/httpjson_false_positive_asserts/changelog.yml b/test/packages/other/httpjson_false_positive_asserts/changelog.yml
new file mode 100644
index 0000000000..1c605437a2
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/changelog.yml
@@ -0,0 +1,5 @@
+- version: "999.999.999"
+  changes:
+    - description: Test assert.hit_count
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/6326
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml
new file mode 100644
index 0000000000..206312f355
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml
@@ -0,0 +1,18 @@
+wait_for_data_timeout: 20s
+input: httpjson
+service: httpjson
+data_stream:
+  vars:
+    data_stream.dataset: httpjson.generic
+    username: test
+    password: test
+    request_interval: 5s
+    request_url: http://{{Hostname}}:{{Port}}/testpagination/api?page=1
+    response_pagination: |-
+      - set:
+          target: url.params.page
+          value: '[[.last_response.body.page]]'
+          fail_on_template_error: true
+    enable_request_tracer: true
+assert:
+  hit_count: 2
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/agent/stream/httpjson.yml.hbs b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/agent/stream/httpjson.yml.hbs
new file mode 100644
index 0000000000..9ce1088341
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/agent/stream/httpjson.yml.hbs
@@ -0,0 +1,161 @@
+config_version: 2
+data_stream:
+  dataset: {{data_stream.dataset}}
+interval: {{request_interval}}
+
+{{#if username}}
+auth.basic.user: {{username}}
+{{/if}}
+{{#if password}}
+auth.basic.password: {{password}}
+{{/if}}
+
+{{#if pipeline}}
+pipeline: {{pipeline}}
+{{/if}}
+
+{{#unless username}}
+{{#unless password}}
+{{#if oauth_id}}
+auth.oauth2.client.id: {{oauth_id}}
+{{/if}}
+{{#if oauth_secret}}
+auth.oauth2.client.secret: {{oauth_secret}}
+{{/if}}
+{{#if oauth_token_url}}
+auth.oauth2.token_url: {{oauth_token_url}}
+{{/if}}
+{{#if oauth_provider}}
+auth.oauth2.provider: {{oauth_provider}}
+{{/if}}
+{{#if oauth_scopes}}
+auth.oauth2.scopes:
+{{#each oauth_scopes as |scope i|}}
+  - {{scope}}
+{{/each}}
+{{/if}}
+{{#if oauth_google_credentials_file}}
+auth.oauth2.google.credentials_file: {{oauth_google_credentials_file}}
+{{/if}}
+{{#if oauth_google_credentials_json}}
+auth.oauth2.google.credentials_json: '{{oauth_google_credentials_json}}'
+{{/if}}
+{{#if oauth_google_jwt_file}}
+auth.oauth2.google.jwt_file: {{oauth_google_jwt_file}}
+{{/if}}
+{{#if oauth_google_jwt_json}}
+auth.oauth2.google.jwt_json: {{oauth_google_jwt_json}}
+{{/if}}
+{{#if oauth_google_delegated_account}}
+auth.oauth2.google.delegated_account: {{oauth_google_delegated_account}}
+{{/if}}
+{{#if oauth_azure_tenant_id}}
+auth.oauth2.azure.tenant_id: {{oauth_azure_tenant_id}}
+{{/if}}
+{{#if oauth_azure_resource}}
+auth.oauth2.azure.resource: {{oauth_azure_resource}}
+{{/if}}
+{{#if oauth_endpoint_params}}
+auth.oauth2.endpoint_params: 
+  {{oauth_endpoint_params}}
+{{/if}}
+{{/unless}}
+{{/unless}}
+
+request.url: {{request_url}}
+request.method: {{request_method}}
+{{#if request_body}}
+request.body: 
+  {{request_body}}
+{{/if}}
+{{#if request_transforms}}
+request.transforms:
+  {{request_transforms}}
+{{/if}}
+{{#if request_ssl}}
+request.ssl: 
+  {{request_ssl}}
+{{/if}}
+{{#if request_encode_as}}
+request.encode_as: {{request_encode_as}}
+{{/if}}
+{{#if request_timeout}}
+request.timeout: {{request_timeout}}
+{{/if}}
+{{#if request_proxy_url}}
+request.proxy_url: {{request_proxy_url}}
+{{/if}}
+{{#if request_retry_max_attempts}}
+request.retry.max_attempts: {{request_retry_max_attempts}}
+{{/if}}
+{{#if request_retry_wait_min}}
+request.retry.wait_min: {{request_retry_wait_min}}
+{{/if}}
+{{#if request_retry_wait_max}}
+request.retry.wait_max: {{request_retry_wait_max}}
+{{/if}}
+{{#if request_redirect_forward_headers}}
+request.redirect.forward_headers: {{request_redirect_forward_headers}}
+{{/if}}
+{{#if request_redirect_headers_ban_list}}
+request.redirect.headers_ban_list:
+{{#each request_redirect_headers_ban_list as |item i|}}
+  - {{item}}
+{{/each}}
+{{/if}}
+{{#if request_redirect_max_redirects}}
+request.redirect.max_redirects: {{request_redirect_max_redirects}}
+{{/if}}
+{{#if request_rate_limit_limit}}
+request.rate_limit.limit: {{request_rate_limit_limit}}
+{{/if}}
+{{#if request_rate_limit_reset}}
+request.rate_limit.reset: {{request_rate_limit_reset}}
+{{/if}}
+{{#if request_rate_limit_remaining}}
+request.rate_limit.remaining: {{request_rate_limit_remaining}}
+{{/if}}
+{{#if enable_request_tracer}}
+request.tracer.filename: "../../logs/httpjson/http-request-trace-*.ndjson"
+{{/if}}
+
+{{#if response_transforms}}
+response.transforms: 
+  {{response_transforms}}
+{{/if}}
+{{#if response_split}}
+response.split: 
+  {{response_split}}
+{{/if}}
+{{#if response_pagination}}
+response.pagination: {{response_pagination}}
+{{/if}}
+{{#if response_decode_as}}
+response.decode_as: {{response_decode_as}}
+{{/if}}
+{{#if response_request_body_on_pagination}}
+response.request_body_on_pagination: {{response_request_body_on_pagination}}
+{{/if}}
+
+{{#if cursor}}
+cursor:
+  {{cursor}}
+{{/if}}
+
+{{#if tags}}
+tags:
+{{#each tags as |tag i|}}
+  - {{tag}}
+{{/each}}
+{{/if}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if chain}}
+chain:
+{{chain}}
+{{/if}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/base-fields.yml b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/base-fields.yml
new file mode 100644
index 0000000000..d8277624ff
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/base-fields.yml
@@ -0,0 +1,20 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: event.module
+  type: constant_keyword
+  description: Event module
+  value: httpjson
+- name: event.dataset
+  type: constant_keyword
+  description: Event dataset
+  value: httpjson.generic
+- name: "@timestamp"
+  type: date
+  description: Event timestamp.
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/beats.yml b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/beats.yml
new file mode 100644
index 0000000000..ede6958855
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/beats.yml
@@ -0,0 +1,6 @@
+- name: input.type
+  description: Type of Filebeat input.
+  type: keyword
+- name: tags
+  type: keyword
+  description: User defined tags
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/ecs.yml b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/ecs.yml
new file mode 100644
index 0000000000..1c3645d5f4
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/ecs.yml
@@ -0,0 +1,6 @@
+- name: ecs.version
+  external: ecs
+- name: event.created
+  external: ecs
+- name: message
+  external: ecs
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/manifest.yml b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/manifest.yml
new file mode 100644
index 0000000000..89b199f86c
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/manifest.yml
@@ -0,0 +1,327 @@
+title: Custom API Input
+type: logs
+streams:
+  - input: httpjson
+    description: Collect custom data from REST API's
+    template_path: httpjson.yml.hbs
+    title: Custom API Input
+    vars:
+      - name: data_stream.dataset
+        type: text
+        title: Dataset name
+        description: |
+          Dataset to write data to. Changing the dataset will send the data to a different index. You can't use `-` in the name of a dataset and only valid characters for [Elasticsearch index names](https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html).
+        default: httpjson.generic
+        required: true
+        show_user: true
+      - name: pipeline
+        type: text
+        title: Ingest Pipeline
+        description: |
+          The Ingest Node pipeline ID to be used by the integration.
+        required: false
+        show_user: true
+      - name: request_url
+        type: text
+        title: Request URL
+        description: i.e. scheme://host:port/path
+        show_user: true
+        required: true
+        default: https://server.example.com:8089/api
+      - name: request_interval
+        type: text
+        title: Request Interval
+        description: How often the API is polled, supports seconds, minutes and hours.
+        show_user: true
+        required: true
+        default: 1m
+      - name: request_method
+        type: text
+        title: Request HTTP Method
+        description: Supports either GET or POST
+        show_user: true
+        required: true
+        default: GET
+      - name: username
+        type: text
+        title: Basic Auth Username
+        show_user: true
+        required: false
+        description: The username to be used with Basic Auth headers
+      - name: password
+        type: password
+        title: Basic Auth Password
+        show_user: true
+        required: false
+        description: The password to be used with Basic Auth headers
+      - name: oauth_id
+        type: text
+        title: Oauth2 Client ID
+        description: Client ID used for Oauth2 authentication
+        show_user: true
+        required: false
+      - name: oauth_secret
+        type: password
+        title: Oauth2 Client Secret
+        description: Client secret used for Oauth2 authentication
+        show_user: true
+        required: false
+      - name: oauth_token_url
+        type: text
+        title: Oauth2 Token URL
+        description: The URL endpoint that will be used to generate the tokens during the oauth2 flow. It is required if no oauth_custom variable is set or provider is not specified in oauth_custom variable.
+        show_user: true
+        required: false
+      - name: request_body
+        type: yaml
+        title: Request Body
+        description: An optional HTTP body if the request method is POST. All available options can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_body)
+        show_user: true
+        multi: false
+        required: false
+      - name: request_transforms
+        type: yaml
+        title: Request Transforms
+        description: Optional transformations to perform on the request before it is sent. All available options can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#request-transforms).
+        show_user: true
+        multi: false
+        required: false
+      - name: response_transforms
+        type: yaml
+        title: Response Transforms
+        description: Optional transformations to perform on the response before it is sent to Elasticsearch. All available options can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#response-transforms).
+        show_user: true
+        multi: false
+        required: false
+      - name: response_split
+        type: yaml
+        title: Response Split
+        description: Optional transformations to perform on the response to split the response into separate documents before it is sent to Elasticsearch. All available options can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#response-split).
+        show_user: true
+        multi: false
+        required: false
+      - name: response_pagination
+        type: yaml
+        title: Response Pagination
+        description: Optional settings if pagination is required to retrieve all results. All available options can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#response-pagination).
+        show_user: true
+        multi: false
+        required: false
+      - name: cursor
+        type: yaml
+        title: Custom request cursor
+        description: |
+          A cursor is used to keep state between each API request, and can be set to for example the value of something in the response body.
+          More information can be found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#cursor).
+        show_user: true
+        multi: false
+        required: false
+      - name: request_ssl
+        type: yaml
+        title: Request SSL Configuration
+        description: i.e. certificate_authorities, supported_protocols, verification_mode etc, more examples found in the [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/configuration-ssl.html#ssl-common-config)
+        multi: false
+        required: false
+        show_user: false
+      - name: request_encode_as
+        type: text
+        title: Request Encode As
+        description: ContentType used for encoding the request body. If set it will force the encoding in the specified format regardless of the Content-Type header value.
+        show_user: false
+        multi: false
+        required: false
+      - name: request_timeout
+        type: text
+        title: Request Timeout
+        description: Duration before declaring that the HTTP client connection has timed out. Valid time units are ns, us, ms, s, m, h. Default is "30"s.
+        show_user: false
+        multi: false
+        required: false
+      - name: request_proxy_url
+        type: text
+        title: Request Proxy
+        description: This specifies proxy configuration in the form of `http[s]://<user>:<password>@<server name/ip>:<port>`.
+        show_user: false
+        multi: false
+        required: false
+      - name: request_retry_max_attempts
+        type: text
+        title: Request Retry Max Attempts
+        description: The maximum number of retries for the HTTP client. Default is "5".
+        show_user: false
+        multi: false
+        required: false
+      - name: request_retry_wait_min
+        type: text
+        title: Request Retry Wait Min
+        description: The minimum time to wait before a retry is attempted. Default is "1s".
+        show_user: false
+        multi: false
+        required: false
+      - name: request_retry_wait_max
+        type: text
+        title: Request Retry Wait Max
+        description: The maximum time to wait before a retry is attempted. Default is "60s".
+        show_user: false
+        multi: false
+        required: false
+      - name: request_redirect_forward_headers
+        type: bool
+        title: Request Redirect Forward Headers
+        description: When set to true request headers are forwarded in case of a redirect. Default is "false".
+        show_user: false
+        multi: false
+        required: false
+      - name: request_redirect_headers_ban_list
+        type: text
+        title: Request Redirect Headers Ban List
+        description: When Redirect Forward Headers is set to true, all headers except the ones defined in this list will be forwarded. All headers are forwarded by default.
+        show_user: false
+        multi: true
+        required: false
+      - name: request_redirect_max_redirects
+        type: text
+        title: Request Redirect Max Redirects
+        description: The maximum number of redirects to follow for a request. Default is "10".
+        show_user: false
+        multi: false
+        required: false
+      - name: request_rate_limit_limit
+        type: text
+        title: Request Rate Limit
+        description: The value of the response that specifies the total limit. It is defined with a Go template value.
+        show_user: false
+        multi: false
+        required: false
+      - name: request_rate_limit_reset
+        type: text
+        title: Request Rate Limit Reset
+        description: The value of the response that specifies the epoch time when the rate limit will reset. It is defined with a Go template value.
+        show_user: false
+        multi: false
+        required: false
+      - name: request_rate_limit_remaining
+        type: text
+        title: Request Rate Limit Remaining
+        description: The value of the response that specifies the remaining quota of the rate limit. It is defined with a Go template value.
+        show_user: false
+        multi: false
+        required: false
+      - name: oauth_provider
+        type: text
+        title: Oauth2 Provider
+        description: Used to configure supported oauth2 providers. Each supported provider will require specific settings. It is not set by default. Supported providers are "azure" and "google".
+        show_user: false
+        multi: false
+        required: false
+      - name: oauth_scopes
+        type: text
+        title: Oauth2 Scopes
+        description: A list of scopes that will be requested during the oauth2 flow. It is optional for all providers.
+        show_user: false
+        multi: true
+        required: false
+      - name: oauth_google_credentials_file
+        type: text
+        title: Oauth2 Google Credentials File
+        description: The full path to the credentials file for Google.
+        show_user: false
+        multi: false
+        required: false
+      - name: oauth_google_credentials_json
+        type: text
+        title: Oauth2 Google Credentials JSON
+        description: Your Google credentials information as raw JSON.
+        show_user: false
+        multi: false
+        required: false
+      - name: oauth_google_jwt_file
+        type: text
+        title: Oauth2 Google JWT File
+        description: Full path to the JWT Account Key file for Google.
+        show_user: false
+        multi: false
+        required: false
+      - name: oauth_google_jwt_json
+        type: text
+        title: Oauth2 Google JWT JSON
+        description: Your Google JWT information as raw JSON.
+        multi: false
+        required: false
+        show_user: false
+      - name: oauth_google_delegated_account
+        type: text
+        title: Oauth2 Google Delegated account
+        description: Email of the delegated account used to create the credentials (usually an admin).
+        show_user: false
+        multi: false
+        required: false
+      - name: oauth_azure_tenant_id
+        type: text
+        title: Oauth2 Azure Tenant ID
+        description: Optional setting used for authentication when using Azure provider. Since it is used in the process to generate the token_url, it can’t be used in combination with it.
+        show_user: false
+        multi: false
+        required: false
+      - name: oauth_azure_resource
+        type: text
+        title: Oauth2 Azure Resource
+        description: Optional setting for the accessed WebAPI resource when using azure provider.
+        show_user: false
+        multi: false
+        required: false
+      - name: oauth_endpoint_params
+        type: yaml
+        title: Oauth2 Endpoint Params
+        description: Set of values that will be sent on each request to the token_url. Each param key can have multiple values. Can be set for all providers except google.
+        show_user: false
+        multi: false
+        required: false
+      - name: response_decode_as
+        type: text
+        title: Response decode settings
+        description: |
+          ContentType used for decoding the response body. Supported values: application/json, application/x-ndjson. By default it will use what is in the response Content-Type header.
+        show_user: false
+        required: false
+      - name: response_request_body_on_pagination
+        type: bool
+        title: Include request body on Pagination
+        description: |
+          If set to true, the values in request.body are sent with pagination requests.
+        show_user: false
+        multi: false
+        required: false
+      - name: chain
+        type: yaml
+        title: Chain
+        multi: false
+        required: false
+        show_user: false
+        description: |
+          A list of requests to be made after the first one. See [`chain`](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#chain) for details.
+      - name: processors
+        type: yaml
+        title: Processors
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details.
+
+      - name: enable_request_tracer
+        type: bool
+        title: Enable request tracing
+        multi: false
+        required: false
+        show_user: false
+        description: >
+          The request tracer logs requests and responses to the agent's local file-system for debugging configurations. Enabling this request tracing compromises security and should only be used for debugging. See [documentation](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-httpjson.html#_request_tracer_filename) for details.
+
+      - name: tags
+        type: text
+        title: Tags
+        multi: true
+        show_user: false
+        default:
+          - forwarded
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/sample_event.json b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/sample_event.json
new file mode 100644
index 0000000000..97f5b56929
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/sample_event.json
@@ -0,0 +1,36 @@
+{
+    "@timestamp": "2022-03-10T12:47:55.098Z",
+    "agent": {
+        "ephemeral_id": "03c96875-43cc-4abc-b998-99527ff31de3",
+        "id": "0ddbfef9-4d38-400d-8404-d2df456bddc0",
+        "name": "docker-fleet-agent",
+        "type": "filebeat",
+        "version": "8.0.0"
+    },
+    "data_stream": {
+        "dataset": "httpjson.generic",
+        "namespace": "ep",
+        "type": "logs"
+    },
+    "ecs": {
+        "version": "8.2.0"
+    },
+    "elastic_agent": {
+        "id": "0ddbfef9-4d38-400d-8404-d2df456bddc0",
+        "snapshot": false,
+        "version": "8.0.0"
+    },
+    "event": {
+        "agent_id_status": "verified",
+        "created": "2022-03-10T12:47:55.098Z",
+        "dataset": "httpjson.generic",
+        "ingested": "2022-03-10T12:47:56Z"
+    },
+    "input": {
+        "type": "httpjson"
+    },
+    "message": "{\"message\":\"success\",\"page\":2}",
+    "tags": [
+        "forwarded"
+    ]
+}
\ No newline at end of file
diff --git a/test/packages/other/httpjson_false_positive_asserts/docs/README.md b/test/packages/other/httpjson_false_positive_asserts/docs/README.md
new file mode 100644
index 0000000000..116fd4803a
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/docs/README.md
@@ -0,0 +1,20 @@
+# Custom API input integration
+
+The custom API input integration is used to ingest data from custom RESTful API's that do not currently have an existing integration.
+
+The input itself supports sending both GET and POST requests, transform requests and responses during runtime, paginate and keep a running state on information from the last collected events.
+
+## Configuration
+
+The extensive documentation for the input are currently available
+
+The most commonly used configuration options are available on the main integration page, while more advanced and customizable options currently resides under the "Advanced options" part of the integration settings page.
+
+Configuration is split into three main categories, Request, Response, and Cursor.
+
+The request part of the configuration handles points like which URL endpoint to communicate with, the request body, specific transformations that have to happen before a request is sent out and some custom options like request proxy, timeout and similar options.
+
+The response part of the configuration handles options like transformation, rate limiting, pagination, and splitting the response into different documents before it is sent to Elasticsearch.
+
+The cursor part of the configuration is used when there is a need to keep state between each of the API requests, for example if a timestamp is returned in the response, that should be used as a filter in the next request after that, the cursor is a place where this is stored.
+
diff --git a/test/packages/other/httpjson_false_positive_asserts/manifest.yml b/test/packages/other/httpjson_false_positive_asserts/manifest.yml
new file mode 100644
index 0000000000..4da9f431ef
--- /dev/null
+++ b/test/packages/other/httpjson_false_positive_asserts/manifest.yml
@@ -0,0 +1,20 @@
+format_version: 2.7.0
+name: httpjson
+title: Custom API
+description: Collect custom events from an API endpoint with Elastic agent
+type: integration
+version: "999.999.999"
+conditions:
+  kibana.version: "^8.7.1"
+categories:
+  - custom
+policy_templates:
+  - name: generic
+    title: Custom API Input
+    description: Collect custom data from REST API's
+    inputs:
+      - type: httpjson
+        title: Collect custom data from REST API's
+        description: Collect custom data from REST API's
+owner:
+  github: elastic/security-external-integrations

From 0a0094f8942a43f247ecf21e9c680c4103120683 Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Fri, 30 Jun 2023 11:31:18 +0200
Subject: [PATCH 04/19] Move the false_positives to own execution

---
 Makefile                                      |  5 +-
 scripts/test-check-false-positives.sh         | 68 +++++++++++++++++++
 scripts/test-check-packages.sh                | 11 +--
 .../_dev/build/build.yml                      |  0
 .../_dev/build/docs/README.md                 |  0
 .../_dev/deploy/docker/docker-compose.yml     |  0
 .../_dev/deploy/docker/files/config.yml       |  0
 .../changelog.yml                             |  0
 .../test/system/test-pagination-config.yml    |  0
 .../generic/agent/stream/httpjson.yml.hbs     |  0
 .../generic/fields/base-fields.yml            |  0
 .../data_stream/generic/fields/beats.yml      |  0
 .../data_stream/generic/fields/ecs.yml        |  0
 .../data_stream/generic/manifest.yml          |  0
 .../data_stream/generic/sample_event.json     |  0
 .../docs/README.md                            |  0
 .../manifest.yml                              |  0
 17 files changed, 73 insertions(+), 11 deletions(-)
 create mode 100755 scripts/test-check-false-positives.sh
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/_dev/build/build.yml (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/_dev/build/docs/README.md (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/changelog.yml (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/data_stream/generic/agent/stream/httpjson.yml.hbs (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/data_stream/generic/fields/base-fields.yml (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/data_stream/generic/fields/beats.yml (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/data_stream/generic/fields/ecs.yml (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/data_stream/generic/manifest.yml (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/data_stream/generic/sample_event.json (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/docs/README.md (100%)
 rename test/packages/{other => false_positives}/httpjson_false_positive_asserts/manifest.yml (100%)

diff --git a/Makefile b/Makefile
index ffa299cc29..2747b84ad6 100644
--- a/Makefile
+++ b/Makefile
@@ -69,7 +69,7 @@ test-stack-command-8x:
 
 test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x
 
-test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks 
+test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-false-positives
 
 test-check-packages-with-kind:
 	PACKAGE_TEST_TYPE=with-kind ./scripts/test-check-packages.sh
@@ -77,6 +77,9 @@ test-check-packages-with-kind:
 test-check-packages-other:
 	PACKAGE_TEST_TYPE=other ./scripts/test-check-packages.sh
 
+test-check-false-positives: 
+	PACKAGE_TEST_TYPE=false_positives ./scripts/test-check-false-positives.sh
+
 test-check-packages-benchmarks:
 	PACKAGE_TEST_TYPE=benchmarks ./scripts/test-check-packages.sh
 
diff --git a/scripts/test-check-false-positives.sh b/scripts/test-check-false-positives.sh
new file mode 100755
index 0000000000..8113f38ee5
--- /dev/null
+++ b/scripts/test-check-false-positives.sh
@@ -0,0 +1,68 @@
+#!/bin/bash
+
+set -euxo pipefail
+
+cleanup() {
+  r=$?
+
+  # Dump stack logs
+  elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-httpjson_false_positive_asserts}}"
+
+  # Take down the stack
+  elastic-package stack down -v
+
+  # Clean used resources
+  for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-httpjson_false_positive_asserts}/; do
+    (
+      cd $d
+      elastic-package clean -v
+    )
+  done
+
+  # This is a false positive scenario and tests that the test case failure is a success scenario  
+  if [ "${PACKAGE_TEST_TYPE:-false_positives}" == "false_positives" ] && [ "${PACKAGE_UNDER_TEST:-httpjson_false_positive_asserts}" == "httpjson_false_positive_asserts" ]; then
+    if [ $r == 1 ]; then
+        exit 0
+      elif [ $r == 0 ]; then
+        exit 1
+      fi
+  fi
+
+  exit $r
+}
+
+trap cleanup EXIT
+
+export ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml"
+
+OLDPWD=$PWD
+# Build/check packages
+for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-httpjson_false_positive_asserts}/; do
+  (
+    cd $d
+    elastic-package check -v
+  )
+done
+cd -
+
+# Update the stack
+elastic-package stack update -v
+
+# Boot up the stack
+elastic-package stack up -d -v
+
+elastic-package stack status
+
+# Run package tests
+eval "$(elastic-package stack shellinit)"
+
+for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-httpjson_false_positive_asserts}/; do
+  (
+    cd $d
+    elastic-package install -v
+
+    # defer-cleanup is set to a short period to verify that the option is available
+    elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage
+  )
+cd -
+done
diff --git a/scripts/test-check-packages.sh b/scripts/test-check-packages.sh
index 5397e5a978..a1c84e894c 100755
--- a/scripts/test-check-packages.sh
+++ b/scripts/test-check-packages.sh
@@ -29,16 +29,7 @@ cleanup() {
       elastic-package clean -v
     )
   done
-
-  # This is a false positive scenario and tests that the test case failure is a success scenario  
-  if [ "${PACKAGE_TEST_TYPE:-other}" == "other" ] && [ "${PACKAGE_UNDER_TEST:-*}" == "httpjson_false_positive_asserts" ]; then
-    if [ $r == 1 ]; then
-        exit 0
-      elif [ $r == 0 ]; then
-        exit 1
-      fi
-  fi
-
+  
   exit $r
 }
 
diff --git a/test/packages/other/httpjson_false_positive_asserts/_dev/build/build.yml b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/build.yml
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/_dev/build/build.yml
rename to test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/build.yml
diff --git a/test/packages/other/httpjson_false_positive_asserts/_dev/build/docs/README.md b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/docs/README.md
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/_dev/build/docs/README.md
rename to test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/docs/README.md
diff --git a/test/packages/other/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml
rename to test/packages/false_positives/httpjson_false_positive_asserts/_dev/deploy/docker/docker-compose.yml
diff --git a/test/packages/other/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml
rename to test/packages/false_positives/httpjson_false_positive_asserts/_dev/deploy/docker/files/config.yml
diff --git a/test/packages/other/httpjson_false_positive_asserts/changelog.yml b/test/packages/false_positives/httpjson_false_positive_asserts/changelog.yml
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/changelog.yml
rename to test/packages/false_positives/httpjson_false_positive_asserts/changelog.yml
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml
rename to test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/_dev/test/system/test-pagination-config.yml
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/agent/stream/httpjson.yml.hbs b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/agent/stream/httpjson.yml.hbs
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/data_stream/generic/agent/stream/httpjson.yml.hbs
rename to test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/agent/stream/httpjson.yml.hbs
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/base-fields.yml b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/base-fields.yml
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/base-fields.yml
rename to test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/base-fields.yml
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/beats.yml b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/beats.yml
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/beats.yml
rename to test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/beats.yml
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/ecs.yml b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/ecs.yml
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/data_stream/generic/fields/ecs.yml
rename to test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/fields/ecs.yml
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/manifest.yml b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/manifest.yml
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/data_stream/generic/manifest.yml
rename to test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/manifest.yml
diff --git a/test/packages/other/httpjson_false_positive_asserts/data_stream/generic/sample_event.json b/test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/sample_event.json
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/data_stream/generic/sample_event.json
rename to test/packages/false_positives/httpjson_false_positive_asserts/data_stream/generic/sample_event.json
diff --git a/test/packages/other/httpjson_false_positive_asserts/docs/README.md b/test/packages/false_positives/httpjson_false_positive_asserts/docs/README.md
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/docs/README.md
rename to test/packages/false_positives/httpjson_false_positive_asserts/docs/README.md
diff --git a/test/packages/other/httpjson_false_positive_asserts/manifest.yml b/test/packages/false_positives/httpjson_false_positive_asserts/manifest.yml
similarity index 100%
rename from test/packages/other/httpjson_false_positive_asserts/manifest.yml
rename to test/packages/false_positives/httpjson_false_positive_asserts/manifest.yml

From f92d3e742ef0d700983b8d4939ce5bf2046e9fce Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Fri, 30 Jun 2023 11:59:11 +0200
Subject: [PATCH 05/19] Add test to buildkite

---
 .buildkite/pipeline.trigger.integration.tests.sh | 1 +
 Makefile                                         | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index c232b6616b..035425d669 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -34,6 +34,7 @@ CHECK_PACKAGES_TESTS=(
     test-check-packages-with-kind
     test-check-packages-with-custom-agent
     test-check-packages-benchmarks
+    test-check-packages-false-positives
 )
 for test in ${CHECK_PACKAGES_TESTS[@]}; do
     echo "      - label: \":go: Running integration test: ${test}\""
diff --git a/Makefile b/Makefile
index 2747b84ad6..3d590f96da 100644
--- a/Makefile
+++ b/Makefile
@@ -69,7 +69,7 @@ test-stack-command-8x:
 
 test-stack-command: test-stack-command-default test-stack-command-7x test-stack-command-800 test-stack-command-8x
 
-test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-false-positives
+test-check-packages: test-check-packages-with-kind test-check-packages-other test-check-packages-parallel test-check-packages-with-custom-agent test-check-packages-benchmarks test-check-packages-false-positives
 
 test-check-packages-with-kind:
 	PACKAGE_TEST_TYPE=with-kind ./scripts/test-check-packages.sh
@@ -77,7 +77,7 @@ test-check-packages-with-kind:
 test-check-packages-other:
 	PACKAGE_TEST_TYPE=other ./scripts/test-check-packages.sh
 
-test-check-false-positives: 
+test-check-packages-false-positives: 
 	PACKAGE_TEST_TYPE=false_positives ./scripts/test-check-false-positives.sh
 
 test-check-packages-benchmarks:

From c91f1820d2e99d8c0f57491d95507dc2e8313bdf Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Mon, 3 Jul 2023 11:57:19 +0200
Subject: [PATCH 06/19] Fix PR comments

---
 internal/testrunner/runners/system/runner.go | 23 ++++++++++++--------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index a8815754cd..cc47cc9182 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -628,16 +628,21 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
 
 		var err error
 		hits, err = r.getDocs(dataStream)
-		if hits.size() == 0 {
-			return false, err
-		}
 
-		ret := hits.size() == oldHits
-		if !ret {
-			oldHits = hits.size()
-		}
+		if config.Assert.HitCount > 0 {
+			if hits.size() == 0 {
+				return false, err
+			}
 
-		return ret, err
+			ret := hits.size() == oldHits
+			if !ret {
+				oldHits = hits.size()
+				time.Sleep(4 * time.Second)
+			}
+
+			return ret, err
+		}
+		return hits.size() > 0, err
 	}, waitForDataTimeout)
 
 	if err != nil {
@@ -987,7 +992,7 @@ func waitUntilTrue(fn func() (bool, error), timeout time.Duration) (bool, error)
 	timeoutTicker := time.NewTicker(timeout)
 	defer timeoutTicker.Stop()
 
-	retryTicker := time.NewTicker(5 * time.Second)
+	retryTicker := time.NewTicker(1 * time.Second)
 	defer retryTicker.Stop()
 
 	for {

From 8bbf9cd8f1476d607404a301da6788c568b2aace Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Mon, 3 Jul 2023 13:42:25 +0200
Subject: [PATCH 07/19] Add separate package run in CI

---
 .buildkite/pipeline.trigger.integration.tests.sh | 16 +++++++++++++++-
 internal/testrunner/runners/system/runner.go     |  4 ++++
 2 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index 035425d669..9bf144627a 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -34,7 +34,6 @@ CHECK_PACKAGES_TESTS=(
     test-check-packages-with-kind
     test-check-packages-with-custom-agent
     test-check-packages-benchmarks
-    test-check-packages-false-positives
 )
 for test in ${CHECK_PACKAGES_TESTS[@]}; do
     echo "      - label: \":go: Running integration test: ${test}\""
@@ -50,6 +49,21 @@ for test in ${CHECK_PACKAGES_TESTS[@]}; do
     fi
 done
 
+pushd test/packages/false_positives > /dev/null
+for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
+    package_name=$(basename ${package})
+    echo " - label: \":go: Running integration test: ${package_name}\""
+    echo " key: \"integration-false_positives-${package_name}\""
+    echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}"
+    echo " agents:" echo " provider: \"gcp\""
+    echo " artifact_paths:"
+    echo " - build/test-results/*.xml"
+    echo " - build/elastic-stack-dump/check-*/logs/*.log"
+    echo " - build/elastic-stack-dump/check-*/logs/fleet-server-internal/**/*"
+ done
+
+ popd > /dev/null
+
 pushd test/packages/parallel > /dev/null
 for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     package_name=$(basename ${package})
diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index cc47cc9182..a77af871ac 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -634,6 +634,10 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
 				return false, err
 			}
 
+			if hits.size() < config.Assert.HitCount {
+				return false, err
+			}
+
 			ret := hits.size() == oldHits
 			if !ret {
 				oldHits = hits.size()

From 6109a0f17ada35485cab623b27336864e8791860 Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <123897612+bhapas@users.noreply.github.com>
Date: Mon, 3 Jul 2023 13:50:36 +0200
Subject: [PATCH 08/19] Remove redundant check

Co-authored-by: Jaime Soriano Pastor <jaime.soriano@elastic.co>
---
 internal/testrunner/runners/system/runner.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/internal/testrunner/runners/system/runner.go b/internal/testrunner/runners/system/runner.go
index a77af871ac..f1af3f0277 100644
--- a/internal/testrunner/runners/system/runner.go
+++ b/internal/testrunner/runners/system/runner.go
@@ -630,10 +630,6 @@ func (r *runner) runTest(config *testConfig, ctxt servicedeployer.ServiceContext
 		hits, err = r.getDocs(dataStream)
 
 		if config.Assert.HitCount > 0 {
-			if hits.size() == 0 {
-				return false, err
-			}
-
 			if hits.size() < config.Assert.HitCount {
 				return false, err
 			}

From 23a011acce4d49b038aa492e27db31862b1c7320 Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Mon, 3 Jul 2023 13:54:47 +0200
Subject: [PATCH 09/19] Fix formatting

---
 .../pipeline.trigger.integration.tests.sh     | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index 9bf144627a..2475e637c9 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -52,15 +52,16 @@ done
 pushd test/packages/false_positives > /dev/null
 for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     package_name=$(basename ${package})
-    echo " - label: \":go: Running integration test: ${package_name}\""
-    echo " key: \"integration-false_positives-${package_name}\""
-    echo " command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}"
-    echo " agents:" echo " provider: \"gcp\""
-    echo " artifact_paths:"
-    echo " - build/test-results/*.xml"
-    echo " - build/elastic-stack-dump/check-*/logs/*.log"
-    echo " - build/elastic-stack-dump/check-*/logs/fleet-server-internal/**/*"
- done
+    echo "      - label: \":go: Running integration test: ${package_name}\""
+    echo "        key: \"integration-false_positives-${package_name}\""
+    echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}"
+    echo "        agents:"
+    echo "          provider: \"gcp\""
+    echo "        artifact_paths:"
+    echo "          - build/test-results/*.xml"
+    echo "          - build/elastic-stack-dump/check-*/logs/*.log"
+    echo "          - build/elastic-stack-dump/check-*/logs/fleet-server-internal/**/*"
+done
 
  popd > /dev/null
 

From 530e9feb404fa06a89e20c09b32a624f88b17b88 Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Mon, 3 Jul 2023 14:41:04 +0200
Subject: [PATCH 10/19] Apply PR comment

---
 .buildkite/pipeline.trigger.integration.tests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index 2475e637c9..e1a7a52172 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -52,7 +52,7 @@ done
 pushd test/packages/false_positives > /dev/null
 for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     package_name=$(basename ${package})
-    echo "      - label: \":go: Running integration test: ${package_name}\""
+    echo "      - label: \":go: Running integration test (false positive): ${package_name}\""
     echo "        key: \"integration-false_positives-${package_name}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}"
     echo "        agents:"

From 0fcce173b7c693a704edfbdbd172b1a527e592a9 Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Mon, 3 Jul 2023 17:32:28 +0200
Subject: [PATCH 11/19] Upload logs to a safe location

---
 .buildkite/pipeline.trigger.integration.tests.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index e1a7a52172..63817d6077 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -55,6 +55,8 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     echo "      - label: \":go: Running integration test (false positive): ${package_name}\""
     echo "        key: \"integration-false_positives-${package_name}\""
     echo "        command: ./.buildkite/scripts/integration_tests.sh -t test-check-packages-false-positives -p ${package_name}"
+    echo "        env:"
+    echo "          UPLOAD_SAFE_LOGS: 1"
     echo "        agents:"
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"

From 4d2190cb08cdfa10994532ca305ff22f997270e7 Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Tue, 4 Jul 2023 07:49:52 +0200
Subject: [PATCH 12/19] Fix script

---
 .buildkite/pipeline.trigger.integration.tests.sh     |  2 --
 scripts/test-check-false-positives.sh                | 12 ++++++------
 .../_dev/build/build.yml                             |  2 +-
 3 files changed, 7 insertions(+), 9 deletions(-)

diff --git a/.buildkite/pipeline.trigger.integration.tests.sh b/.buildkite/pipeline.trigger.integration.tests.sh
index 63817d6077..744c8a506c 100755
--- a/.buildkite/pipeline.trigger.integration.tests.sh
+++ b/.buildkite/pipeline.trigger.integration.tests.sh
@@ -61,8 +61,6 @@ for package in $(find . -maxdepth 1 -mindepth 1 -type d) ; do
     echo "          provider: \"gcp\""
     echo "        artifact_paths:"
     echo "          - build/test-results/*.xml"
-    echo "          - build/elastic-stack-dump/check-*/logs/*.log"
-    echo "          - build/elastic-stack-dump/check-*/logs/fleet-server-internal/**/*"
 done
 
  popd > /dev/null
diff --git a/scripts/test-check-false-positives.sh b/scripts/test-check-false-positives.sh
index 8113f38ee5..97334b8873 100755
--- a/scripts/test-check-false-positives.sh
+++ b/scripts/test-check-false-positives.sh
@@ -6,13 +6,13 @@ cleanup() {
   r=$?
 
   # Dump stack logs
-  elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-httpjson_false_positive_asserts}}"
+  elastic-package stack dump -v --output "build/elastic-stack-dump/check-${PACKAGE_UNDER_TEST:-${PACKAGE_TEST_TYPE:-*}}"
 
   # Take down the stack
   elastic-package stack down -v
 
   # Clean used resources
-  for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-httpjson_false_positive_asserts}/; do
+  for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do
     (
       cd $d
       elastic-package clean -v
@@ -20,12 +20,12 @@ cleanup() {
   done
 
   # This is a false positive scenario and tests that the test case failure is a success scenario  
-  if [ "${PACKAGE_TEST_TYPE:-false_positives}" == "false_positives" ] && [ "${PACKAGE_UNDER_TEST:-httpjson_false_positive_asserts}" == "httpjson_false_positive_asserts" ]; then
+  if [ "${PACKAGE_TEST_TYPE:-false_positives}" == "false_positives" ]; then
     if [ $r == 1 ]; then
         exit 0
       elif [ $r == 0 ]; then
         exit 1
-      fi
+    fi
   fi
 
   exit $r
@@ -37,7 +37,7 @@ export ELASTIC_PACKAGE_LINKS_FILE_PATH="$(pwd)/scripts/links_table.yml"
 
 OLDPWD=$PWD
 # Build/check packages
-for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-httpjson_false_positive_asserts}/; do
+for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do
   (
     cd $d
     elastic-package check -v
@@ -56,7 +56,7 @@ elastic-package stack status
 # Run package tests
 eval "$(elastic-package stack shellinit)"
 
-for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-httpjson_false_positive_asserts}/; do
+for d in test/packages/${PACKAGE_TEST_TYPE:-false_positives}/${PACKAGE_UNDER_TEST:-*}/; do
   (
     cd $d
     elastic-package install -v
diff --git a/test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/build.yml b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/build.yml
index 875463aaf4..074278e5b1 100644
--- a/test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/build.yml
+++ b/test/packages/false_positives/httpjson_false_positive_asserts/_dev/build/build.yml
@@ -1,3 +1,3 @@
 dependencies:
   ecs:
-    reference: git@8.8
+    reference: git@v8.8.0

From 68fb07a189c3326b103971792b5e0111f074a0f9 Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Tue, 4 Jul 2023 10:08:28 +0200
Subject: [PATCH 13/19] Modify false positive test execution target

---
 .buildkite/scripts/integration_tests.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.buildkite/scripts/integration_tests.sh b/.buildkite/scripts/integration_tests.sh
index fb1637c117..1ef5a3d0b1 100755
--- a/.buildkite/scripts/integration_tests.sh
+++ b/.buildkite/scripts/integration_tests.sh
@@ -24,6 +24,7 @@ source .buildkite/scripts/install_deps.sh
 source .buildkite/scripts/tooling.sh
 
 PARALLEL_TARGET="test-check-packages-parallel"
+FALSE_POSITIVES_TARGET="test-check-packages-false-positives"
 KIND_TARGET="test-check-packages-with-kind"
 TMP_FOLDER_TEMPLATE="${TMP_FOLDER_TEMPLATE_BASE}.XXXXXXXXX"
 GOOGLE_CREDENTIALS_FILENAME="google-cloud-credentials.json"
@@ -108,7 +109,7 @@ if [[ "${TARGET}" == "${KIND_TARGET}" ]]; then
 fi
 
 echo "--- Run integration test ${TARGET}"
-if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]]; then
+if [[ "${TARGET}" == "${PARALLEL_TARGET}" ]] || [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then
     make install
 
     # allow to fail this command, to be able to upload safe logs

From 8deca247dd1fe11ff47669f06dfd6ae61f3ebd28 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Tue, 4 Jul 2023 11:44:29 +0200
Subject: [PATCH 14/19] Add integration-false_positives as step for GCS
 credentials

---
 .buildkite/hooks/pre-command | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command
index 8dec4bff9f..97ccf5b29b 100644
--- a/.buildkite/hooks/pre-command
+++ b/.buildkite/hooks/pre-command
@@ -33,7 +33,7 @@ PRIVATE_CI_GCS_CREDENTIALS_PATH=kv/ci-shared/platform-ingest/private_ci_artifact
 # Secrets must be redacted
 # https://buildkite.com/docs/pipelines/managing-log-output#redacted-environment-variables
 
-if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && "$BUILDKITE_STEP_KEY" =~ ^integration-parallel ]]; then
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-package" && ("$BUILDKITE_STEP_KEY" =~ ^integration-parallel || "$BUILDKITE_STEP_KEY" =~ ^integration-false_positives) ]]; then
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET=$(retry 5 vault kv get -field plaintext ${PRIVATE_CI_GCS_CREDENTIALS_PATH})
 fi
 

From d4867993c5c6224d701a2ca2495674c7cd23b5f6 Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <123897612+bhapas@users.noreply.github.com>
Date: Wed, 5 Jul 2023 05:50:07 +0200
Subject: [PATCH 15/19] Update scripts/test-check-false-positives.sh

Co-authored-by: Mario Rodriguez Molins <marrodmo@gmail.com>
---
 scripts/test-check-false-positives.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/scripts/test-check-false-positives.sh b/scripts/test-check-false-positives.sh
index 97334b8873..62a730306c 100755
--- a/scripts/test-check-false-positives.sh
+++ b/scripts/test-check-false-positives.sh
@@ -22,9 +22,11 @@ cleanup() {
   # This is a false positive scenario and tests that the test case failure is a success scenario  
   if [ "${PACKAGE_TEST_TYPE:-false_positives}" == "false_positives" ]; then
     if [ $r == 1 ]; then
-        exit 0
-      elif [ $r == 0 ]; then
-        exit 1
+      rm build/test-results/*.xml
+      exit 0
+    elif [ $r == 0 ]; then
+      echo "Expected to fail tests, but there was none failing"
+      exit 1
     fi
   fi
 

From 51e33f4d8e93fc18b7eae030911167fb0f150f55 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 5 Jul 2023 11:45:00 +0200
Subject: [PATCH 16/19] Check for expected errors in false_positives script

---
 Makefile                                      |  2 +-
 scripts/test-check-false-positives.sh         | 27 ++++++++++++++++---
 ...son_false_positive_asserts.expected_errors |  2 ++
 3 files changed, 27 insertions(+), 4 deletions(-)
 create mode 100644 test/packages/false_positives/httpjson_false_positive_asserts.expected_errors

diff --git a/Makefile b/Makefile
index 3d590f96da..d9656e54f4 100644
--- a/Makefile
+++ b/Makefile
@@ -77,7 +77,7 @@ test-check-packages-with-kind:
 test-check-packages-other:
 	PACKAGE_TEST_TYPE=other ./scripts/test-check-packages.sh
 
-test-check-packages-false-positives: 
+test-check-packages-false-positives:
 	PACKAGE_TEST_TYPE=false_positives ./scripts/test-check-false-positives.sh
 
 test-check-packages-benchmarks:
diff --git a/scripts/test-check-false-positives.sh b/scripts/test-check-false-positives.sh
index 62a730306c..d8e82ea5da 100755
--- a/scripts/test-check-false-positives.sh
+++ b/scripts/test-check-false-positives.sh
@@ -19,13 +19,34 @@ cleanup() {
     )
   done
 
-  # This is a false positive scenario and tests that the test case failure is a success scenario  
+  # This is a false positive scenario and tests that the test case failure is a success scenario
   if [ "${PACKAGE_TEST_TYPE:-false_positives}" == "false_positives" ]; then
     if [ $r == 1 ]; then
-      rm build/test-results/*.xml
+      EXPECTED_ERRORS_FILE="test/packages/false_positives/${PACKAGE_UNDER_TEST}.expected_errors"
+      if [ ! -f ${EXPECTED_ERRORS_FILE} ]; then
+        echo "Error: Missing expected errors file: ${EXPECTED_ERRORS_FILE}"
+      fi
+      RESULTS_NO_SPACES="build/test-results-no-spaces.xml"
+      cat build/test-results/*.xml | tr -d '\n' > ${RESULTS_NO_SPACES}
+
+      # check number of expected errors
+      number_errors=$(cat build/test-results/*.xml | grep "<failure>" | wc -l)
+      expected_errors=$(cat ${EXPECTED_ERRORS_FILE} | wc -l)
+
+      if [ ${number_errors} -ne ${expected_errors} ]; then
+          echo "Error: There are unexpected errors in ${PACKAGE_UNDER_TEST}"
+          exit 1
+      fi
+
+      # check whether or not the expected errors exist in the xml files
+      while read -r line; do
+        cat ${RESULTS_NO_SPACES} | grep -E "${line}"
+      done < ${EXPECTED_ERRORS_FILE}
+      rm -f build/test-results/*.xml
+      rm -f ${RESULTS_NO_SPACES}
       exit 0
     elif [ $r == 0 ]; then
-      echo "Expected to fail tests, but there was none failing"
+      echo "Error: Expected to fail tests, but there was none failing"
       exit 1
     fi
   fi
diff --git a/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors b/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors
new file mode 100644
index 0000000000..68adcf628f
--- /dev/null
+++ b/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors
@@ -0,0 +1,2 @@
+<testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failure>
+<testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failure>

From 08933fe0eac04accb5ab211738a8bd570061baf1 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 5 Jul 2023 12:07:49 +0200
Subject: [PATCH 17/19] Try with same number but different error

---
 .../httpjson_false_positive_asserts.expected_errors            | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors b/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors
index 68adcf628f..ffa80439be 100644
--- a/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors
+++ b/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors
@@ -1,2 +1 @@
-<testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failure>
-<testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failure>
+<testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failur>

From 30bf3b00507c441a1552b022eca63b13e82c0020 Mon Sep 17 00:00:00 2001
From: Mario Rodriguez Molins <mario.rodriguez@elastic.co>
Date: Wed, 5 Jul 2023 12:57:51 +0200
Subject: [PATCH 18/19] Set the right expected errors

---
 .../httpjson_false_positive_asserts.expected_errors             | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors b/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors
index ffa80439be..4cb323f578 100644
--- a/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors
+++ b/test/packages/false_positives/httpjson_false_positive_asserts.expected_errors
@@ -1 +1 @@
-<testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failur>
+<testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failure>

From c2d77cb95cb8ee1af4fd6f2a9fb927faf08ad8f2 Mon Sep 17 00:00:00 2001
From: Bharat Pasupula <saibharatchandra.pasupula@elastic.co>
Date: Wed, 5 Jul 2023 14:43:12 +0200
Subject: [PATCH 19/19] Add negative system tests to documentation

---
 docs/howto/system_testing.md | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/docs/howto/system_testing.md b/docs/howto/system_testing.md
index 95f7312a5a..c14a6f4961 100644
--- a/docs/howto/system_testing.md
+++ b/docs/howto/system_testing.md
@@ -568,6 +568,23 @@ to indexing generated data from the integration's data streams into Elasticsearc
 elastic-package test system --generate
 ```
 
+### System testing negative or false-positive scenarios
+
+The system tests support packages to be tested for negative scenarios. An example would be to test that the `assert.hit_count` is verified when all the docs are ingested rather than just finding enough docs for the testcase.
+
+There are some special rules for testing negative scenarios
+
+- The negative / false-positive test packages are added under `test/packages/false_positives`
+- It is required to have a file `<package_name>.expected_errors` with the lines needed for every package added under `test/packages/false_positives`.
+- One line per error, taking into account that all `\n` were removed, meaning it is just one line for everything.
+- As it is used `grep` with `-E` some kind of regexes can be used.
+
+Example `expected_errors` file content:
+
+```xml
+<testcase name=\"system test: pagination\" classname=\"httpjson_false_positive_asserts.generic\" time=\".*\"> * <failure>observed hit count 4 did not match expected hit count 2</failure>
+```
+
 ## Continuous Integration
 
 `elastic-package` runs a set of system tests on some [dummy packages](https://github.com/elastic/elastic-package/tree/main/test/packages) to ensure it's functionalities work as expected. This allows to test changes affecting package testing within `elastic-package` before merging and releasing the changes.