From 8ae93e8c410573d93f4532f17079935c331159ea Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Fri, 14 Jul 2023 00:10:35 +0200 Subject: [PATCH 1/7] Fix oldest API query param handling --- packages/slack/changelog.yml | 5 +++++ .../audit/_dev/test/system/test-default-config.yml | 1 + .../slack/data_stream/audit/agent/stream/httpjson.yml.hbs | 8 ++++---- packages/slack/manifest.yml | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/slack/changelog.yml b/packages/slack/changelog.yml index 1f2fa54bfd56..7be4afaf12a0 100644 --- a/packages/slack/changelog.yml +++ b/packages/slack/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.7.1" + changes: + - description: Fix handling of API query param 'oldest' + type: bugfix + link: https://github.com/elastic/integrations/pull/6958 - version: "1.7.0" changes: - description: Document valid duration units. diff --git a/packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml b/packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml index 1b2a2193af6d..b0fd58ed5319 100644 --- a/packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml +++ b/packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml @@ -9,5 +9,6 @@ data_stream: preserve_original_event: true oauth_token: xoxp-1234567890 limit: 2 + initial_interval: 4800h assert: hit_count: 4 diff --git a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs index 8be41fc30677..951ecfda6ff1 100644 --- a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs +++ b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs @@ -21,11 +21,11 @@ request.transforms: value: "Bearer {{oauth_token}}" - set: target: url.params.oldest - value: "[[.cursor.last_timestamp]]" - default: '[[(now (parseDuration "-{{initial_interval}}")).Unix]]' + value: '[[(now (parseDuration "-{{initial_interval}}")).Unix]]' - set: target: url.params.latest - value: '[[(now).Unix]]' + default: '[[(now).Unix]]' + value: "[[.cursor.last_timestamp]]" - set: target: url.params.limit value: '[[{{limit}}]]' @@ -43,7 +43,7 @@ response.pagination: cursor: last_timestamp: - value: "[[toInt .first_event.date_create]]" + value: "[[toInt .last_event.date_create]]" fail_on_template_error: true {{#if tags.length}} diff --git a/packages/slack/manifest.yml b/packages/slack/manifest.yml index e699671cd42a..8a4179da7256 100644 --- a/packages/slack/manifest.yml +++ b/packages/slack/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: slack title: "Slack Logs" -version: "1.7.0" +version: "1.7.1" license: basic release: ga description: "Slack Logs Integration" From dbfa822d8cf5a63026f4a9ed61fe98d1c8a48e95 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Thu, 27 Jul 2023 13:53:17 +0200 Subject: [PATCH 2/7] Remove oldest and latest params in pagination request --- .../slack/data_stream/audit/agent/stream/httpjson.yml.hbs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs index 951ecfda6ff1..53bb19cdc3a9 100644 --- a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs +++ b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs @@ -40,6 +40,11 @@ response.pagination: target: url.params.cursor value: '[[.last_response.body.response_metadata.next_cursor]]' fail_on_template_error: true + +- delete: + target: url.params.oldest +- delete: + target: url.params.latest cursor: last_timestamp: From 3cadfeb9b93af1897ae2a42a4008d59316dc46f0 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Mon, 31 Jul 2023 12:19:56 +0200 Subject: [PATCH 3/7] Fix the agent config --- .../slack/_dev/deploy/docker/files/config.yml | 2 - .../audit/agent/stream/httpjson.yml.hbs | 40 ++++++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/packages/slack/_dev/deploy/docker/files/config.yml b/packages/slack/_dev/deploy/docker/files/config.yml index 99868d5f55ec..2939a87759d6 100644 --- a/packages/slack/_dev/deploy/docker/files/config.yml +++ b/packages/slack/_dev/deploy/docker/files/config.yml @@ -5,8 +5,6 @@ rules: authorization: Bearer xoxp-1234567890 accept: application/json query_params: - oldest: "{oldest:\\d+}" - latest: "{latest:\\d+}" limit: "2" cursor: YXNkZmFzZGZhc2Rm responses: diff --git a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs index 53bb19cdc3a9..5492fb38b4f5 100644 --- a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs +++ b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs @@ -21,11 +21,32 @@ request.transforms: value: "Bearer {{oauth_token}}" - set: target: url.params.oldest - value: '[[(now (parseDuration "-{{initial_interval}}")).Unix]]' + # If last pagination cycle was finished successully + # we move the oldest pointer forward + # else we reprocess the last interval from the beginning. + # If none of the values are in the cursor it means is a fresh start + # and we pick the default value. + value: >- + [[- if eq .cursor.pagination_finished "true" -]] + [[- .cursor.next_oldest_date -]] + [[- else -]] + [[- .cursor.last_oldest_date -]] + [[- end -]] + default: '[[(now (parseDuration "-{{initial_interval}}")).Unix]]' - set: target: url.params.latest + # If last pagination cycle was finished successully + # we move the latest pointer forward + # else we reprocess the last interval from the beginning. + # If none of the values are in the cursor it means is a fresh start + # and we pick the default value. + value: >- + [[- if eq .cursor.pagination_finished "true" -]] + [[(now).Unix]] + [[- else -]] + [[- .cursor.last_latest_date -]] + [[- end -]] default: '[[(now).Unix]]' - value: "[[.cursor.last_timestamp]]" - set: target: url.params.limit value: '[[{{limit}}]]' @@ -47,6 +68,21 @@ response.pagination: target: url.params.latest cursor: + pagination_finished: + # Use this flag to identify if an execution was interrupted in the middle + # of a pagination cycle. + value: '[[not (index .last_response.body.response_metadata "next_cursor")]]' + last_oldest_date: + # Use this value to be able to resume from an interrupted pagination cycle. + value: '[[.last_request.url.params.Get "oldest"]]' + last_latest_date: + # Use this value to be able to resume from an interrupted pagination cycle. + value: '[[.last_request.url.params.Get "latest"]]' + +# in order to pick the next startDate we keep the first event (newest) time. + next_oldest_date: + value: "[[toInt .first_event.date_create]]" + fail_on_template_error: true last_timestamp: value: "[[toInt .last_event.date_create]]" fail_on_template_error: true From 230858553b4481b3c1988e4093befbb0f6825e2e Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Tue, 1 Aug 2023 11:45:57 +0200 Subject: [PATCH 4/7] Address pr comments --- .../audit/agent/stream/httpjson.yml.hbs | 56 ++++++++----------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs index 5492fb38b4f5..af6e14e517f0 100644 --- a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs +++ b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs @@ -19,34 +19,25 @@ request.transforms: - set: target: header.Authorization value: "Bearer {{oauth_token}}" + + # If the pagination is interrupted, then the last cursor + # is saved and sent in the request url to continue pagination. + # If pagination is complete, then the new request will have 'first_event' + # date as 'oldest' param and 'now' as 'latest' param + {{#if this.cursor.pagination_incomplete}} + - set: + target: url.params.cursor + value: '[[- .cursor.last_cursor -]]' + {{else}} - set: target: url.params.oldest - # If last pagination cycle was finished successully - # we move the oldest pointer forward - # else we reprocess the last interval from the beginning. - # If none of the values are in the cursor it means is a fresh start - # and we pick the default value. - value: >- - [[- if eq .cursor.pagination_finished "true" -]] - [[- .cursor.next_oldest_date -]] - [[- else -]] - [[- .cursor.last_oldest_date -]] - [[- end -]] + value: '[[- .cursor.next_oldest_date -]]' default: '[[(now (parseDuration "-{{initial_interval}}")).Unix]]' - set: target: url.params.latest - # If last pagination cycle was finished successully - # we move the latest pointer forward - # else we reprocess the last interval from the beginning. - # If none of the values are in the cursor it means is a fresh start - # and we pick the default value. - value: >- - [[- if eq .cursor.pagination_finished "true" -]] - [[(now).Unix]] - [[- else -]] - [[- .cursor.last_latest_date -]] - [[- end -]] - default: '[[(now).Unix]]' + value: '[[(now).Unix]]' + {{/if}} + - set: target: url.params.limit value: '[[{{limit}}]]' @@ -68,24 +59,21 @@ response.pagination: target: url.params.latest cursor: - pagination_finished: + pagination_incomplete: # Use this flag to identify if an execution was interrupted in the middle # of a pagination cycle. - value: '[[not (index .last_response.body.response_metadata "next_cursor")]]' - last_oldest_date: - # Use this value to be able to resume from an interrupted pagination cycle. - value: '[[.last_request.url.params.Get "oldest"]]' - last_latest_date: + value: '[[(index .last_response.body.response_metadata "next_cursor")]]' + fail_on_template_error: true + + last_cursor: # Use this value to be able to resume from an interrupted pagination cycle. - value: '[[.last_request.url.params.Get "latest"]]' + value: '[[.last_response.body.response_metadata.Get "next_cursor"]]' + fail_on_template_error: true -# in order to pick the next startDate we keep the first event (newest) time. next_oldest_date: + # In order to pick the next startDate we keep the first event (newest) date. value: "[[toInt .first_event.date_create]]" fail_on_template_error: true - last_timestamp: - value: "[[toInt .last_event.date_create]]" - fail_on_template_error: true {{#if tags.length}} tags: From 6644e1fb480d8054c97b4bad46ac8663dd453a76 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Thu, 3 Aug 2023 13:15:29 +0200 Subject: [PATCH 5/7] Modify agent --- .../audit/agent/stream/httpjson.yml.hbs | 42 +++++++++++++------ 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs index af6e14e517f0..4400b497c64c 100644 --- a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs +++ b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs @@ -20,23 +20,40 @@ request.transforms: target: header.Authorization value: "Bearer {{oauth_token}}" - # If the pagination is interrupted, then the last cursor - # is saved and sent in the request url to continue pagination. - # If pagination is complete, then the new request will have 'first_event' - # date as 'oldest' param and 'now' as 'latest' param - {{#if this.cursor.pagination_incomplete}} - set: target: url.params.cursor value: '[[- .cursor.last_cursor -]]' - {{else}} + default: '' + - set: target: url.params.oldest - value: '[[- .cursor.next_oldest_date -]]' + # If last pagination cycle was finished successully + # we move the oldest pointer forward + # else we continue processing using cursor + # If none of the values are in the cursor it means is a fresh start + # and we pick the default value. + value: >- + [[- if eq .cursor.pagination_finished "true" -]] + [[- .cursor.next_oldest_date -]] + [[- else -]] + '' + [[- end -]] default: '[[(now (parseDuration "-{{initial_interval}}")).Unix]]' + - set: target: url.params.latest - value: '[[(now).Unix]]' - {{/if}} + # If last pagination cycle was finished successully + # we move the latest pointer forward + # else we continue processing using cursor + # If none of the values are in the cursor it means is a fresh start + # and we pick the default value. + value: >- + [[- if eq .cursor.pagination_finished "true" -]] + [[(now).Unix]] + [[- else -]] + '' + [[- end -]] + default: '[[(now).Unix]]' - set: target: url.params.limit @@ -59,12 +76,11 @@ response.pagination: target: url.params.latest cursor: - pagination_incomplete: + pagination_finished: # Use this flag to identify if an execution was interrupted in the middle # of a pagination cycle. - value: '[[(index .last_response.body.response_metadata "next_cursor")]]' - fail_on_template_error: true - + value: '[[not (index .last_response.body.response_metadata "next_cursor")]]' + last_cursor: # Use this value to be able to resume from an interrupted pagination cycle. value: '[[.last_response.body.response_metadata.Get "next_cursor"]]' From ca5e5217a15eb95f743dbbd9879cbca995abcd90 Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Fri, 4 Aug 2023 11:23:43 +0200 Subject: [PATCH 6/7] Fix agent config and added system test --- .../slack/_dev/deploy/docker/files/config.yml | 69 +++++++++++++++++-- .../_dev/test/system/test-default-config.yml | 3 +- .../audit/agent/stream/httpjson.yml.hbs | 53 +++++--------- 3 files changed, 83 insertions(+), 42 deletions(-) diff --git a/packages/slack/_dev/deploy/docker/files/config.yml b/packages/slack/_dev/deploy/docker/files/config.yml index 2939a87759d6..7a7f8ab04166 100644 --- a/packages/slack/_dev/deploy/docker/files/config.yml +++ b/packages/slack/_dev/deploy/docker/files/config.yml @@ -1,4 +1,61 @@ rules: + - path: /audit/v1/logs + methods: ["GET"] + request_headers: + authorization: Bearer xoxp-1234567890 + accept: application/json + query_params: + oldest: "1683836285" + limit: "2" + responses: + - status_code: 200 + body: |- + { + "entries":[ + {"action":"file_downloaded","actor":{"type":"user","user":{"email":"user.mcuser@abcd.co","id":"2f53269c-4f38-4f58-b56d-c2b968681dbd","name":"User McUser","team":"user-team"}},"context":{"ip_address":"81.2.69.144","location":{"domain":"domain.tld","id":"eedd1a7d-1a92-418d-8b01-51a4c809d0fb","name":"The Place","type":"workspace"},"session_id":913888259765,"ua":"com.tinyspeck.chatlyio/23.04.40 (iPhone; iOS 1.4.1; Scale/3.00)"},"date_create":1683836290,"details":{"url_private":"https://example.com/"},"entity":{"file":{"filetype":"image/png","id":"7edc4c42-f925-47af-979a-22c10e1fefed","name":"image.png","title":"image.png"},"type":"file"},"id":"2db28080-1659-4b27-ad55-fdba12e3a7b1"}, + {"id":"1665fc41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1683836291,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}} + ] + } + - path: /audit/v1/logs + methods: ["GET"] + request_headers: + authorization: Bearer xoxp-1234567890 + accept: application/json + query_params: + limit: "2" + cursor: GytjmKHF5hFmty + responses: + - status_code: 200 + body: |- + { + "entries":[ + {"action":"file_downloaded","actor":{"type":"user","user":{"email":"user.mcuser@abcd.co","id":"2f53269c-4f38-4f08-b56d-c2b968681dbd","name":"User McUser","team":"user-team"}},"context":{"ip_address":"81.2.69.144","location":{"domain":"domain.tld","id":"eedd1a7d-1a92-418d-8b01-51a4c809d0fb","name":"The Place","type":"workspace"},"session_id":913888259765,"ua":"com.tinyspeck.chatlyio/23.04.40 (iPhone; iOS 1.4.1; Scale/3.00)"},"date_create":1683836285,"details":{"url_private":"https://example.com/"},"entity":{"file":{"filetype":"image/png","id":"7edc4c42-f925-47af-979a-22c10e1fefed","name":"image.png","title":"image.png"},"type":"file"},"id":"2db28070-1659-4b27-ad55-fdba12e3a7b1"}, + {"id":"1665fb41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1683836281,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}} + ], + "response_metadata": { + "next_cursor": "" + } + } + - path: /audit/v1/logs + methods: ["GET"] + request_headers: + authorization: Bearer xoxp-1234567890 + accept: application/json + query_params: + oldest: "1683836277" + limit: "2" + responses: + - status_code: 200 + body: |- + { + "entries":[ + {"id":"bdcd13e3-28a3-41f0-9ace-a20952def3a0","date_create":1683836285,"action":"user_created","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"181.2.69.143"}}, + {"id":"0126a45b-6c7d-8900-e12f-3456789gh0i1","date_create":1683836287,"action":"user_login","actor":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"entity":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"context":{"location":{"type":"enterprise","id":"E1701NCCA","name":"Birdland","domain":"birdland"},"ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36","ip_address":"81.2.69.143"}} + ], + "response_metadata": { + "next_cursor": "GytjmKHF5hFmty" + } + } - path: /audit/v1/logs methods: ["GET"] request_headers: @@ -13,8 +70,11 @@ rules: { "entries":[ {"action":"file_downloaded","actor":{"type":"user","user":{"email":"user.mcuser@abcd.co","id":"2f52269c-4f38-4f08-b56d-c2b968681dbd","name":"User McUser","team":"user-team"}},"context":{"ip_address":"81.2.69.144","location":{"domain":"domain.tld","id":"eedd1a7d-1a92-418d-8b01-51a4c809d0fb","name":"The Place","type":"workspace"},"session_id":913888259765,"ua":"com.tinyspeck.chatlyio/23.04.40 (iPhone; iOS 1.4.1; Scale/3.00)"},"date_create":1683836275,"details":{"url_private":"https://example.com/"},"entity":{"file":{"filetype":"image/png","id":"7edc4c42-f925-47af-979a-22c10e1fefed","name":"image.png","title":"image.png"},"type":"file"},"id":"2db28060-1659-4b27-ad55-fdba12e3a7b1"}, - {"id":"16f5fb41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1673631531,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}} - ] + {"id":"16f5fb41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1683836271,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}} + ], + "response_metadata": { + "next_cursor": "" + } } - path: /audit/v1/logs methods: ["GET"] @@ -23,15 +83,14 @@ rules: accept: application/json query_params: oldest: "{oldest:\\d+}" - latest: "{latest:\\d+}" limit: "2" responses: - status_code: 200 body: |- { "entries":[ - {"id":"bdcb13e3-28a3-41f0-9ace-a20952def3a0","date_create":1566215192,"action":"user_created","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"181.2.69.143"}}, - {"id":"0123a45b-6c7d-8900-e12f-3456789gh0i1","date_create":1521214343,"action":"user_login","actor":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"entity":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"context":{"location":{"type":"enterprise","id":"E1701NCCA","name":"Birdland","domain":"birdland"},"ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36","ip_address":"81.2.69.143"}} + {"id":"bdcb13e3-28a3-41f0-9ace-a20952def3a0","date_create":1683836277,"action":"user_created","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"181.2.69.143"}}, + {"id":"0123a45b-6c7d-8900-e12f-3456789gh0i1","date_create":1683836279,"action":"user_login","actor":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"entity":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"context":{"location":{"type":"enterprise","id":"E1701NCCA","name":"Birdland","domain":"birdland"},"ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36","ip_address":"81.2.69.143"}} ], "response_metadata": { "next_cursor": "YXNkZmFzZGZhc2Rm" diff --git a/packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml b/packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml index b0fd58ed5319..47a38decac15 100644 --- a/packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml +++ b/packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml @@ -10,5 +10,6 @@ data_stream: oauth_token: xoxp-1234567890 limit: 2 initial_interval: 4800h + interval: 10s assert: - hit_count: 4 + hit_count: 10 diff --git a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs index 4400b497c64c..a6196dd72171 100644 --- a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs +++ b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs @@ -22,7 +22,9 @@ request.transforms: - set: target: url.params.cursor - value: '[[- .cursor.last_cursor -]]' + value: '[[.cursor.next_cursor]]' + # An empty value omits this param from being set on the request. + # This should be omitted if it is a new request or a request after pagination is complete. default: '' - set: @@ -32,29 +34,12 @@ request.transforms: # else we continue processing using cursor # If none of the values are in the cursor it means is a fresh start # and we pick the default value. - value: >- - [[- if eq .cursor.pagination_finished "true" -]] + value: |- + [[- if .cursor.pagination_finished -]] [[- .cursor.next_oldest_date -]] - [[- else -]] - '' [[- end -]] default: '[[(now (parseDuration "-{{initial_interval}}")).Unix]]' - - set: - target: url.params.latest - # If last pagination cycle was finished successully - # we move the latest pointer forward - # else we continue processing using cursor - # If none of the values are in the cursor it means is a fresh start - # and we pick the default value. - value: >- - [[- if eq .cursor.pagination_finished "true" -]] - [[(now).Unix]] - [[- else -]] - '' - [[- end -]] - default: '[[(now).Unix]]' - - set: target: url.params.limit value: '[[{{limit}}]]' @@ -65,31 +50,27 @@ request.rate_limit.remaining: '0' # hardcoded to 0 since slack doesn't return re response.split: target: body.entries response.pagination: -- set: - target: url.params.cursor - value: '[[.last_response.body.response_metadata.next_cursor]]' - fail_on_template_error: true - -- delete: - target: url.params.oldest -- delete: - target: url.params.latest + - set: + target: url.params.cursor + value: '[[.last_response.body.response_metadata.next_cursor]]' + fail_on_template_error: true + - delete: + target: url.params.oldest cursor: pagination_finished: # Use this flag to identify if an execution was interrupted in the middle # of a pagination cycle. - value: '[[not (index .last_response.body.response_metadata "next_cursor")]]' - - last_cursor: + # For completed pagination next_cursor is "" and if the response is not paginated + # then the blob response_metadata does not exist + value: '[[and (not (index .last_response.body "response_metadata")) (eq (.last_response.body.response_metadata.Get "next_cursor") "")]]' + next_cursor: # Use this value to be able to resume from an interrupted pagination cycle. value: '[[.last_response.body.response_metadata.Get "next_cursor"]]' - fail_on_template_error: true - next_oldest_date: # In order to pick the next startDate we keep the first event (newest) date. - value: "[[toInt .first_event.date_create]]" - fail_on_template_error: true + value: '[[toInt .first_event.date_create]]' + default: '[[(now).Unix]]' {{#if tags.length}} tags: From 3d98a222c3357f1316be7bff4e67d3045314ea0d Mon Sep 17 00:00:00 2001 From: Bharat Pasupula Date: Mon, 7 Aug 2023 09:27:43 +0200 Subject: [PATCH 7/7] Remove default in next_oldest_date --- packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs index a6196dd72171..c6217f363731 100644 --- a/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs +++ b/packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs @@ -70,7 +70,6 @@ cursor: next_oldest_date: # In order to pick the next startDate we keep the first event (newest) date. value: '[[toInt .first_event.date_create]]' - default: '[[(now).Unix]]' {{#if tags.length}} tags: