[slack] Fix 'oldest' API query param handling

packages/slack/_dev/deploy/docker/files/config.yml

@@ -5,8 +5,6 @@ rules:
       authorization: Bearer xoxp-1234567890
       accept: application/json
     query_params:
-      oldest: "{oldest:\\d+}"
-      latest: "{latest:\\d+}"
       limit: "2"
       cursor: YXNkZmFzZGZhc2Rm
     responses:

packages/slack/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.9.1"
+  changes:
+    - description: Fix handling of API query param 'oldest'
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/6958
 - version: "1.9.0"
   changes:
     - description: Update package to ECS 8.9.0.

packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml

@@ -9,5 +9,6 @@ data_stream:
     preserve_original_event: true
     oauth_token: xoxp-1234567890
     limit: 2
+    initial_interval: 4800h
 assert:
   hit_count: 4

packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs

@@ -19,13 +19,25 @@ request.transforms:
   - set:
       target: header.Authorization
       value: "Bearer {{oauth_token}}"
+
+  # If the pagination is interrupted, then the last cursor
+  # is saved and sent in the request url to continue pagination.
+  # If pagination is complete, then the new request will have 'first_event'
+  # date as 'oldest' param and 'now' as 'latest' param 
+  {{#if this.cursor.pagination_incomplete}}
+  - set:
+      target: url.params.cursor
+      value: '[[- .cursor.last_cursor -]]'
+  {{else}}
   - set:
       target: url.params.oldest
-      value: "[[.cursor.last_timestamp]]"
+      value: '[[- .cursor.next_oldest_date -]]'
       default: '[[(now (parseDuration "-{{initial_interval}}")).Unix]]'
   - set:
       target: url.params.latest
       value: '[[(now).Unix]]'
+  {{/if}}
+
   - set:
       target: url.params.limit
       value: '[[{{limit}}]]'
@@ -40,9 +52,26 @@ response.pagination:
     target: url.params.cursor
     value: '[[.last_response.body.response_metadata.next_cursor]]'
     fail_on_template_error: true
+
+- delete:
+    target: url.params.oldest
+- delete:
+    target: url.params.latest
 
 cursor:
-  last_timestamp:
+  pagination_incomplete:
+    # Use this flag to identify if an execution was interrupted in the middle
+    # of a pagination cycle.
+    value: '[[(index .last_response.body.response_metadata "next_cursor")]]'
+    fail_on_template_error: true
+
+  last_cursor:
+    # Use this value to be able to resume from an interrupted pagination cycle.
+    value: '[[.last_response.body.response_metadata.Get "next_cursor"]]'
+    fail_on_template_error: true
+
+  next_oldest_date:
+    # In order to pick the next startDate we keep the first event (newest) date.
     value: "[[toInt .first_event.date_create]]"
     fail_on_template_error: true
 

packages/slack/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: slack
 title: "Slack Logs"
-version: "1.9.0"
+version: "1.9.1"
 license: basic
 release: ga
 description: "Slack Logs Integration"