From e0722812561a09d4516c966b30ae4f1bea2cd984 Mon Sep 17 00:00:00 2001
From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com>
Date: Fri, 9 Feb 2024 12:42:59 +0100
Subject: [PATCH] [8.12](backport #37898) [filebeat][threatintel] MISP
 pagination fixes (#37924)

[filebeat][threatintel] MISP pagination fixes (#37898)

Update the HTTP JSON input configuration for the Threat Intel module's
misp fileset with pagination fixes that were done earlier in the
Agent-based MISP integration, in these PRs:

- Fix timestamp format sent to API
  https://github.com/elastic/integrations/pull/6482

- Fix duplicate requests for page 1
  https://github.com/elastic/integrations/pull/6495

- Keep the same timestamp for later pages
  https://github.com/elastic/integrations/pull/6649

- Pagination fixes
  https://github.com/elastic/integrations/pull/9073
---
 CHANGELOG.next.asciidoc                       |  1 +
 .../module/threatintel/misp/config/config.yml | 25 ++++++++++++++++---
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index a3ecc6b2b01..40628a1d6d4 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -47,6 +47,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 
 *Filebeat*
 
+- [threatintel] MISP pagination fixes {pull}37898[37898]
 
 *Heartbeat*
 
diff --git a/x-pack/filebeat/module/threatintel/misp/config/config.yml b/x-pack/filebeat/module/threatintel/misp/config/config.yml
index 3bd5aac30ec..9ad66efcf54 100644
--- a/x-pack/filebeat/module/threatintel/misp/config/config.yml
+++ b/x-pack/filebeat/module/threatintel/misp/config/config.yml
@@ -32,8 +32,20 @@ request.transforms:
     value: json
 - set:
     target: body.timestamp
-    value: '[[.cursor.timestamp]]'
-    default: '[[ formatDate (now (parseDuration "-{{ .first_interval }}")) "UnixDate" ]]'
+    value: >-
+      [[- if index .cursor "timestamp" -]]
+        [[- .cursor.timestamp -]]
+      [[- else -]]
+        [[- .last_response.url.params.Get "timestamp" -]]
+      [[- end -]]
+    default: '[[ (now (parseDuration "-{{ .first_interval }}")).Unix ]]'
+- set:
+    target: body.order
+    value: timestamp
+- set:
+    # Ignored by MISP, set as a workaround to make it available in response.pagination.
+    target: url.params.timestamp
+    value: '[[.body.timestamp]]'
 
 response.split:
   target: body.response
@@ -51,8 +63,15 @@ response.request_body_on_pagination: true
 response.pagination:
 - set:
     target: body.page
-    value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 1]][[end]]'
+    # Add 2 because the httpjson page counter is zero-based while the MISP page parameter starts at 1.
+    value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 2]][[end]]'
     fail_on_template_error: true
+- set:
+    target: body.timestamp
+    value: '[[.last_response.url.params.Get "timestamp"]]'
+- set:
+    target: url.params.timestamp
+    value: '[[.last_response.url.params.Get "timestamp"]]'
 cursor:
   timestamp:
     value: '[[.last_event.Event.timestamp]]'