[slack] Fix 'oldest' API query param handling

packages/slack/_dev/deploy/docker/files/config.yml

@@ -5,8 +5,63 @@ rules:
       authorization: Bearer xoxp-1234567890
       accept: application/json
     query_params:
-      oldest: "{oldest:\\d+}"
-      latest: "{latest:\\d+}"
+      oldest: "1683836285"
+      limit: "2"
+    responses:
+      - status_code: 200
+        body: |-
+          {
+            "entries":[
+              {"action":"file_downloaded","actor":{"type":"user","user":{"email":"user.mcuser@abcd.co","id":"2f53269c-4f38-4f58-b56d-c2b968681dbd","name":"User McUser","team":"user-team"}},"context":{"ip_address":"81.2.69.144","location":{"domain":"domain.tld","id":"eedd1a7d-1a92-418d-8b01-51a4c809d0fb","name":"The Place","type":"workspace"},"session_id":913888259765,"ua":"com.tinyspeck.chatlyio/23.04.40 (iPhone; iOS 1.4.1; Scale/3.00)"},"date_create":1683836290,"details":{"url_private":"https://example.com/"},"entity":{"file":{"filetype":"image/png","id":"7edc4c42-f925-47af-979a-22c10e1fefed","name":"image.png","title":"image.png"},"type":"file"},"id":"2db28080-1659-4b27-ad55-fdba12e3a7b1"},
+              {"id":"1665fc41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1683836291,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}}
+            ]
+          }
+  - path: /audit/v1/logs
+    methods: ["GET"]
+    request_headers:
+      authorization: Bearer xoxp-1234567890
+      accept: application/json
+    query_params:
+      limit: "2"
+      cursor: GytjmKHF5hFmty
+    responses:
+      - status_code: 200
+        body: |-
+          {
+            "entries":[
+              {"action":"file_downloaded","actor":{"type":"user","user":{"email":"user.mcuser@abcd.co","id":"2f53269c-4f38-4f08-b56d-c2b968681dbd","name":"User McUser","team":"user-team"}},"context":{"ip_address":"81.2.69.144","location":{"domain":"domain.tld","id":"eedd1a7d-1a92-418d-8b01-51a4c809d0fb","name":"The Place","type":"workspace"},"session_id":913888259765,"ua":"com.tinyspeck.chatlyio/23.04.40 (iPhone; iOS 1.4.1; Scale/3.00)"},"date_create":1683836285,"details":{"url_private":"https://example.com/"},"entity":{"file":{"filetype":"image/png","id":"7edc4c42-f925-47af-979a-22c10e1fefed","name":"image.png","title":"image.png"},"type":"file"},"id":"2db28070-1659-4b27-ad55-fdba12e3a7b1"},
+              {"id":"1665fb41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1683836281,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}}
+            ],
+            "response_metadata": {
+                "next_cursor": ""
+            }
+          }
+  - path: /audit/v1/logs
+    methods: ["GET"]
+    request_headers:
+      authorization: Bearer xoxp-1234567890
+      accept: application/json
+    query_params:
+      oldest: "1683836277"
+      limit: "2"
+    responses:
+      - status_code: 200
+        body: |-
+          {
+            "entries":[
+              {"id":"bdcd13e3-28a3-41f0-9ace-a20952def3a0","date_create":1683836285,"action":"user_created","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"181.2.69.143"}},
+              {"id":"0126a45b-6c7d-8900-e12f-3456789gh0i1","date_create":1683836287,"action":"user_login","actor":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"entity":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"context":{"location":{"type":"enterprise","id":"E1701NCCA","name":"Birdland","domain":"birdland"},"ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36","ip_address":"81.2.69.143"}}
+            ],
+            "response_metadata": {
+                "next_cursor": "GytjmKHF5hFmty"
+            }
+          }
+  - path: /audit/v1/logs
+    methods: ["GET"]
+    request_headers:
+      authorization: Bearer xoxp-1234567890
+      accept: application/json
+    query_params:
       limit: "2"
       cursor: YXNkZmFzZGZhc2Rm
     responses:
@@ -15,8 +70,11 @@ rules:
           {
             "entries":[
               {"action":"file_downloaded","actor":{"type":"user","user":{"email":"user.mcuser@abcd.co","id":"2f52269c-4f38-4f08-b56d-c2b968681dbd","name":"User McUser","team":"user-team"}},"context":{"ip_address":"81.2.69.144","location":{"domain":"domain.tld","id":"eedd1a7d-1a92-418d-8b01-51a4c809d0fb","name":"The Place","type":"workspace"},"session_id":913888259765,"ua":"com.tinyspeck.chatlyio/23.04.40 (iPhone; iOS 1.4.1; Scale/3.00)"},"date_create":1683836275,"details":{"url_private":"https://example.com/"},"entity":{"file":{"filetype":"image/png","id":"7edc4c42-f925-47af-979a-22c10e1fefed","name":"image.png","title":"image.png"},"type":"file"},"id":"2db28060-1659-4b27-ad55-fdba12e3a7b1"},
-              {"id":"16f5fb41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1673631531,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}}
-            ]
+              {"id":"16f5fb41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1683836271,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}}
+            ],
+            "response_metadata": {
+                "next_cursor": ""
+            }
           }
   - path: /audit/v1/logs
     methods: ["GET"]
@@ -25,15 +83,14 @@ rules:
       accept: application/json
     query_params:
       oldest: "{oldest:\\d+}"
-      latest: "{latest:\\d+}"
       limit: "2"
     responses:
       - status_code: 200
         body: |-
           {
             "entries":[
-              {"id":"bdcb13e3-28a3-41f0-9ace-a20952def3a0","date_create":1566215192,"action":"user_created","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"181.2.69.143"}},
-              {"id":"0123a45b-6c7d-8900-e12f-3456789gh0i1","date_create":1521214343,"action":"user_login","actor":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"entity":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"context":{"location":{"type":"enterprise","id":"E1701NCCA","name":"Birdland","domain":"birdland"},"ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36","ip_address":"81.2.69.143"}}
+              {"id":"bdcb13e3-28a3-41f0-9ace-a20952def3a0","date_create":1683836277,"action":"user_created","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"181.2.69.143"}},
+              {"id":"0123a45b-6c7d-8900-e12f-3456789gh0i1","date_create":1683836279,"action":"user_login","actor":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"entity":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"context":{"location":{"type":"enterprise","id":"E1701NCCA","name":"Birdland","domain":"birdland"},"ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36","ip_address":"81.2.69.143"}}
             ],
             "response_metadata": {
                 "next_cursor": "YXNkZmFzZGZhc2Rm"

packages/slack/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.9.1"
+  changes:
+    - description: Fix handling of API query param 'oldest'
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/6958
 - version: "1.9.0"
   changes:
     - description: Update package to ECS 8.9.0.

packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml

@@ -9,5 +9,7 @@ data_stream:
     preserve_original_event: true
     oauth_token: xoxp-1234567890
     limit: 2
+    initial_interval: 4800h
+    interval: 10s
 assert:
-  hit_count: 4
+  hit_count: 10

packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs

@@ -19,13 +19,27 @@ request.transforms:
   - set:
       target: header.Authorization
       value: "Bearer {{oauth_token}}"
+
+  - set:
+      target: url.params.cursor
+      value: '[[.cursor.next_cursor]]'
+      # An empty value omits this param from being set on the request.
+      # This should be omitted if it is a new request or a request after pagination is complete.
+      default: ''
+
   - set:
       target: url.params.oldest
-      value: "[[.cursor.last_timestamp]]"
+      # If last pagination cycle was finished successully
+      # we move the oldest pointer forward
+      # else we continue processing using cursor
+      # If none of the values are in the cursor it means is a fresh start
+      # and we pick the default value.
+      value: |-
+        [[- if .cursor.pagination_finished -]]
+          [[- .cursor.next_oldest_date -]]
+        [[- end -]]
       default: '[[(now (parseDuration "-{{initial_interval}}")).Unix]]'
-  - set:
-      target: url.params.latest
-      value: '[[(now).Unix]]'
+
   - set:
       target: url.params.limit
       value: '[[{{limit}}]]'
@@ -36,15 +50,27 @@ request.rate_limit.remaining: '0' # hardcoded to 0 since slack doesn't return re
 response.split:
   target: body.entries
 response.pagination:
-- set:
-    target: url.params.cursor
-    value: '[[.last_response.body.response_metadata.next_cursor]]'
-    fail_on_template_error: true
+  - set:
+      target: url.params.cursor
+      value: '[[.last_response.body.response_metadata.next_cursor]]'
+      fail_on_template_error: true
+  - delete:
+      target: url.params.oldest
 
 cursor:
-  last_timestamp:
-    value: "[[toInt .first_event.date_create]]"
-    fail_on_template_error: true
+  pagination_finished:
+    # Use this flag to identify if an execution was interrupted in the middle
+    # of a pagination cycle.
+    # For completed pagination next_cursor is "" and if the response is not paginated
+    # then the blob response_metadata does not exist
+    value: '[[and (not (index .last_response.body "response_metadata")) (eq (.last_response.body.response_metadata.Get "next_cursor") "")]]'
+  next_cursor:
+    # Use this value to be able to resume from an interrupted pagination cycle.
+    value: '[[.last_response.body.response_metadata.Get "next_cursor"]]'
+  next_oldest_date:
+    # In order to pick the next startDate we keep the first event (newest) date.
+    value: '[[toInt .first_event.date_create]]'
+    default: '[[(now).Unix]]'
 
 {{#if tags.length}}
 tags:

packages/slack/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: slack
 title: "Slack Logs"
-version: "1.9.0"
+version: "1.9.1"
 license: basic
 release: ga
 description: "Slack Logs Integration"