From dd22a9683ec5cf766f77ef16c380eeea3880e456 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula <123897612+bhapas@users.noreply.github.com> Date: Wed, 9 Aug 2023 16:31:41 +0200 Subject: [PATCH] [slack] Fix 'oldest' API query param handling (#6958) * Fix oldest API query param handling * Remove oldest and latest params in pagination request * Fix the agent config * Address pr comments * Modify agent * Fix agent config and added system test * Remove default in next_oldest_date --- .../slack/_dev/deploy/docker/files/config.yml | 71 +++++++++++++++++-- packages/slack/changelog.yml | 5 ++ .../_dev/test/system/test-default-config.yml | 4 +- .../audit/agent/stream/httpjson.yml.hbs | 47 +++++++++--- packages/slack/manifest.yml | 2 +- 5 files changed, 109 insertions(+), 20 deletions(-) diff --git a/packages/slack/_dev/deploy/docker/files/config.yml b/packages/slack/_dev/deploy/docker/files/config.yml index 99868d5f55ec..7a7f8ab04166 100644 --- a/packages/slack/_dev/deploy/docker/files/config.yml +++ b/packages/slack/_dev/deploy/docker/files/config.yml @@ -5,8 +5,63 @@ rules: authorization: Bearer xoxp-1234567890 accept: application/json query_params: - oldest: "{oldest:\\d+}" - latest: "{latest:\\d+}" + oldest: "1683836285" + limit: "2" + responses: + - status_code: 200 + body: |- + { + "entries":[ + {"action":"file_downloaded","actor":{"type":"user","user":{"email":"user.mcuser@abcd.co","id":"2f53269c-4f38-4f58-b56d-c2b968681dbd","name":"User McUser","team":"user-team"}},"context":{"ip_address":"81.2.69.144","location":{"domain":"domain.tld","id":"eedd1a7d-1a92-418d-8b01-51a4c809d0fb","name":"The Place","type":"workspace"},"session_id":913888259765,"ua":"com.tinyspeck.chatlyio/23.04.40 (iPhone; iOS 1.4.1; Scale/3.00)"},"date_create":1683836290,"details":{"url_private":"https://example.com/"},"entity":{"file":{"filetype":"image/png","id":"7edc4c42-f925-47af-979a-22c10e1fefed","name":"image.png","title":"image.png"},"type":"file"},"id":"2db28080-1659-4b27-ad55-fdba12e3a7b1"}, + {"id":"1665fc41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1683836291,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}} + ] + } + - path: /audit/v1/logs + methods: ["GET"] + request_headers: + authorization: Bearer xoxp-1234567890 + accept: application/json + query_params: + limit: "2" + cursor: GytjmKHF5hFmty + responses: + - status_code: 200 + body: |- + { + "entries":[ + {"action":"file_downloaded","actor":{"type":"user","user":{"email":"user.mcuser@abcd.co","id":"2f53269c-4f38-4f08-b56d-c2b968681dbd","name":"User McUser","team":"user-team"}},"context":{"ip_address":"81.2.69.144","location":{"domain":"domain.tld","id":"eedd1a7d-1a92-418d-8b01-51a4c809d0fb","name":"The Place","type":"workspace"},"session_id":913888259765,"ua":"com.tinyspeck.chatlyio/23.04.40 (iPhone; iOS 1.4.1; Scale/3.00)"},"date_create":1683836285,"details":{"url_private":"https://example.com/"},"entity":{"file":{"filetype":"image/png","id":"7edc4c42-f925-47af-979a-22c10e1fefed","name":"image.png","title":"image.png"},"type":"file"},"id":"2db28070-1659-4b27-ad55-fdba12e3a7b1"}, + {"id":"1665fb41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1683836281,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}} + ], + "response_metadata": { + "next_cursor": "" + } + } + - path: /audit/v1/logs + methods: ["GET"] + request_headers: + authorization: Bearer xoxp-1234567890 + accept: application/json + query_params: + oldest: "1683836277" + limit: "2" + responses: + - status_code: 200 + body: |- + { + "entries":[ + {"id":"bdcd13e3-28a3-41f0-9ace-a20952def3a0","date_create":1683836285,"action":"user_created","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"181.2.69.143"}}, + {"id":"0126a45b-6c7d-8900-e12f-3456789gh0i1","date_create":1683836287,"action":"user_login","actor":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"entity":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"context":{"location":{"type":"enterprise","id":"E1701NCCA","name":"Birdland","domain":"birdland"},"ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36","ip_address":"81.2.69.143"}} + ], + "response_metadata": { + "next_cursor": "GytjmKHF5hFmty" + } + } + - path: /audit/v1/logs + methods: ["GET"] + request_headers: + authorization: Bearer xoxp-1234567890 + accept: application/json + query_params: limit: "2" cursor: YXNkZmFzZGZhc2Rm responses: @@ -15,8 +70,11 @@ rules: { "entries":[ {"action":"file_downloaded","actor":{"type":"user","user":{"email":"user.mcuser@abcd.co","id":"2f52269c-4f38-4f08-b56d-c2b968681dbd","name":"User McUser","team":"user-team"}},"context":{"ip_address":"81.2.69.144","location":{"domain":"domain.tld","id":"eedd1a7d-1a92-418d-8b01-51a4c809d0fb","name":"The Place","type":"workspace"},"session_id":913888259765,"ua":"com.tinyspeck.chatlyio/23.04.40 (iPhone; iOS 1.4.1; Scale/3.00)"},"date_create":1683836275,"details":{"url_private":"https://example.com/"},"entity":{"file":{"filetype":"image/png","id":"7edc4c42-f925-47af-979a-22c10e1fefed","name":"image.png","title":"image.png"},"type":"file"},"id":"2db28060-1659-4b27-ad55-fdba12e3a7b1"}, - {"id":"16f5fb41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1673631531,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}} - ] + {"id":"16f5fb41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1683836271,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}} + ], + "response_metadata": { + "next_cursor": "" + } } - path: /audit/v1/logs methods: ["GET"] @@ -25,15 +83,14 @@ rules: accept: application/json query_params: oldest: "{oldest:\\d+}" - latest: "{latest:\\d+}" limit: "2" responses: - status_code: 200 body: |- { "entries":[ - {"id":"bdcb13e3-28a3-41f0-9ace-a20952def3a0","date_create":1566215192,"action":"user_created","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"181.2.69.143"}}, - {"id":"0123a45b-6c7d-8900-e12f-3456789gh0i1","date_create":1521214343,"action":"user_login","actor":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"entity":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"context":{"location":{"type":"enterprise","id":"E1701NCCA","name":"Birdland","domain":"birdland"},"ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36","ip_address":"81.2.69.143"}} + {"id":"bdcb13e3-28a3-41f0-9ace-a20952def3a0","date_create":1683836277,"action":"user_created","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"181.2.69.143"}}, + {"id":"0123a45b-6c7d-8900-e12f-3456789gh0i1","date_create":1683836279,"action":"user_login","actor":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"entity":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"context":{"location":{"type":"enterprise","id":"E1701NCCA","name":"Birdland","domain":"birdland"},"ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36","ip_address":"81.2.69.143"}} ], "response_metadata": { "next_cursor": "YXNkZmFzZGZhc2Rm" diff --git a/packages/slack/changelog.yml b/packages/slack/changelog.yml index ae0627560d2f..ecb6072f976f 100644 --- a/packages/slack/changelog.yml +++ b/packages/slack/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.10.1" + changes: + - description: Fix handling of API query param 'oldest' + type: bugfix + link: https://github.com/elastic/integrations/pull/6958 - version: "1.10.0" changes: - description: Update package-spec to 2.9.0. diff --git a/packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml b/packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml index 1b2a2193af6d..47a38decac15 100644 --- a/packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml +++ b/packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml @@ -9,5 +9,7 @@ data_stream: preserve_original_event: true oauth_token: xoxp-1234567890 limit: 2 + initial_interval: 4800h + interval: 10s assert: - hit_count: 4 + hit_count: 10 diff --git a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs index 8be41fc30677..c6217f363731 100644 --- a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs +++ b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs @@ -19,13 +19,27 @@ request.transforms: - set: target: header.Authorization value: "Bearer {{oauth_token}}" + + - set: + target: url.params.cursor + value: '[[.cursor.next_cursor]]' + # An empty value omits this param from being set on the request. + # This should be omitted if it is a new request or a request after pagination is complete. + default: '' + - set: target: url.params.oldest - value: "[[.cursor.last_timestamp]]" + # If last pagination cycle was finished successully + # we move the oldest pointer forward + # else we continue processing using cursor + # If none of the values are in the cursor it means is a fresh start + # and we pick the default value. + value: |- + [[- if .cursor.pagination_finished -]] + [[- .cursor.next_oldest_date -]] + [[- end -]] default: '[[(now (parseDuration "-{{initial_interval}}")).Unix]]' - - set: - target: url.params.latest - value: '[[(now).Unix]]' + - set: target: url.params.limit value: '[[{{limit}}]]' @@ -36,15 +50,26 @@ request.rate_limit.remaining: '0' # hardcoded to 0 since slack doesn't return re response.split: target: body.entries response.pagination: -- set: - target: url.params.cursor - value: '[[.last_response.body.response_metadata.next_cursor]]' - fail_on_template_error: true + - set: + target: url.params.cursor + value: '[[.last_response.body.response_metadata.next_cursor]]' + fail_on_template_error: true + - delete: + target: url.params.oldest cursor: - last_timestamp: - value: "[[toInt .first_event.date_create]]" - fail_on_template_error: true + pagination_finished: + # Use this flag to identify if an execution was interrupted in the middle + # of a pagination cycle. + # For completed pagination next_cursor is "" and if the response is not paginated + # then the blob response_metadata does not exist + value: '[[and (not (index .last_response.body "response_metadata")) (eq (.last_response.body.response_metadata.Get "next_cursor") "")]]' + next_cursor: + # Use this value to be able to resume from an interrupted pagination cycle. + value: '[[.last_response.body.response_metadata.Get "next_cursor"]]' + next_oldest_date: + # In order to pick the next startDate we keep the first event (newest) date. + value: '[[toInt .first_event.date_create]]' {{#if tags.length}} tags: diff --git a/packages/slack/manifest.yml b/packages/slack/manifest.yml index b49672dd79f5..58fe12f9f68d 100644 --- a/packages/slack/manifest.yml +++ b/packages/slack/manifest.yml @@ -1,7 +1,7 @@ format_version: 2.9.0 name: slack title: "Slack Logs" -version: "1.10.0" +version: "1.10.1" description: "Slack Logs Integration" type: integration categories: