From b7fc69a3ee3211300be9432a622a2ff0d26f45ff Mon Sep 17 00:00:00 2001
From: Chris Berkhout <chris.berkhout@elastic.co>
Date: Thu, 8 Feb 2024 10:08:15 +0100
Subject: [PATCH] [filebeat][threatintel] MISP pagination fixes (#37898)

Update the HTTP JSON input configuration for the Threat Intel module's
misp fileset with pagination fixes that were done earlier in the
Agent-based MISP integration, in these PRs:

- Fix timestamp format sent to API
  https://github.com/elastic/integrations/pull/6482

- Fix duplicate requests for page 1
  https://github.com/elastic/integrations/pull/6495

- Keep the same timestamp for later pages
  https://github.com/elastic/integrations/pull/6649

- Pagination fixes
  https://github.com/elastic/integrations/pull/9073
---
 CHANGELOG.next.asciidoc                       |  1 +
 .../module/threatintel/misp/config/config.yml | 25 ++++++++++++++++---
 2 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 8281f7b79ec..5c9d49a5e1c 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -89,6 +89,7 @@ fields added to events containing the Beats version. {pull}37553[37553]
 - Fix m365_defender cursor value and query building. {pull}37116[37116]
 - Fix TCP/UDP metric queue length parsing base. {pull}37714[37714]
 - Update github.com/lestrrat-go/jwx dependency. {pull}37799[37799]
+- [threatintel] MISP pagination fixes {pull}37898[37898]
 
 *Heartbeat*
 
diff --git a/x-pack/filebeat/module/threatintel/misp/config/config.yml b/x-pack/filebeat/module/threatintel/misp/config/config.yml
index 3bd5aac30ec..9ad66efcf54 100644
--- a/x-pack/filebeat/module/threatintel/misp/config/config.yml
+++ b/x-pack/filebeat/module/threatintel/misp/config/config.yml
@@ -32,8 +32,20 @@ request.transforms:
     value: json
 - set:
     target: body.timestamp
-    value: '[[.cursor.timestamp]]'
-    default: '[[ formatDate (now (parseDuration "-{{ .first_interval }}")) "UnixDate" ]]'
+    value: >-
+      [[- if index .cursor "timestamp" -]]
+        [[- .cursor.timestamp -]]
+      [[- else -]]
+        [[- .last_response.url.params.Get "timestamp" -]]
+      [[- end -]]
+    default: '[[ (now (parseDuration "-{{ .first_interval }}")).Unix ]]'
+- set:
+    target: body.order
+    value: timestamp
+- set:
+    # Ignored by MISP, set as a workaround to make it available in response.pagination.
+    target: url.params.timestamp
+    value: '[[.body.timestamp]]'
 
 response.split:
   target: body.response
@@ -51,8 +63,15 @@ response.request_body_on_pagination: true
 response.pagination:
 - set:
     target: body.page
-    value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 1]][[end]]'
+    # Add 2 because the httpjson page counter is zero-based while the MISP page parameter starts at 1.
+    value: '[[if (ne (len .last_response.body.response) 0)]][[add .last_response.page 2]][[end]]'
     fail_on_template_error: true
+- set:
+    target: body.timestamp
+    value: '[[.last_response.url.params.Get "timestamp"]]'
+- set:
+    target: url.params.timestamp
+    value: '[[.last_response.url.params.Get "timestamp"]]'
 cursor:
   timestamp:
     value: '[[.last_event.Event.timestamp]]'