Skip to content

Commit

Permalink
[slack] Fix 'oldest' API query param handling (#6958)
Browse files Browse the repository at this point in the history 
* Fix oldest API query param handling

* Remove oldest and latest params in pagination request

* Fix the agent config

* Address pr comments

* Modify agent

* Fix agent config and added system test

* Remove default in next_oldest_date
  • Loading branch information
@bhapas
bhapas committed Aug 9, 2023
1 parent 4786ff9 commit ab4d3a1
Show file tree
Hide file tree
Showing 5 changed files with 109 additions and 20 deletions.
71 changes: 64 additions & 7 deletions packages/slack/_dev/deploy/docker/files/config.yml
View file
Expand Up @@ -5,8 +5,63 @@ rules:
authorization: Bearer xoxp-1234567890
accept: application/json
query_params:
oldest: "{oldest:\\d+}"
latest: "{latest:\\d+}"
oldest: "1683836285"
limit: "2"
responses:
- status_code: 200
body: |-
{
"entries":[
{"action":"file_downloaded","actor":{"type":"user","user":{"email":"user.mcuser@abcd.co","id":"2f53269c-4f38-4f58-b56d-c2b968681dbd","name":"User McUser","team":"user-team"}},"context":{"ip_address":"81.2.69.144","location":{"domain":"domain.tld","id":"eedd1a7d-1a92-418d-8b01-51a4c809d0fb","name":"The Place","type":"workspace"},"session_id":913888259765,"ua":"com.tinyspeck.chatlyio/23.04.40 (iPhone; iOS 1.4.1; Scale/3.00)"},"date_create":1683836290,"details":{"url_private":"https://example.com/"},"entity":{"file":{"filetype":"image/png","id":"7edc4c42-f925-47af-979a-22c10e1fefed","name":"image.png","title":"image.png"},"type":"file"},"id":"2db28080-1659-4b27-ad55-fdba12e3a7b1"},
{"id":"1665fc41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1683836291,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}}
]
}
- path: /audit/v1/logs
methods: ["GET"]
request_headers:
authorization: Bearer xoxp-1234567890
accept: application/json
query_params:
limit: "2"
cursor: GytjmKHF5hFmty
responses:
- status_code: 200
body: |-
{
"entries":[
{"action":"file_downloaded","actor":{"type":"user","user":{"email":"user.mcuser@abcd.co","id":"2f53269c-4f38-4f08-b56d-c2b968681dbd","name":"User McUser","team":"user-team"}},"context":{"ip_address":"81.2.69.144","location":{"domain":"domain.tld","id":"eedd1a7d-1a92-418d-8b01-51a4c809d0fb","name":"The Place","type":"workspace"},"session_id":913888259765,"ua":"com.tinyspeck.chatlyio/23.04.40 (iPhone; iOS 1.4.1; Scale/3.00)"},"date_create":1683836285,"details":{"url_private":"https://example.com/"},"entity":{"file":{"filetype":"image/png","id":"7edc4c42-f925-47af-979a-22c10e1fefed","name":"image.png","title":"image.png"},"type":"file"},"id":"2db28070-1659-4b27-ad55-fdba12e3a7b1"},
{"id":"1665fb41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1683836281,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}}
],
"response_metadata": {
"next_cursor": ""
}
}
- path: /audit/v1/logs
methods: ["GET"]
request_headers:
authorization: Bearer xoxp-1234567890
accept: application/json
query_params:
oldest: "1683836277"
limit: "2"
responses:
- status_code: 200
body: |-
{
"entries":[
{"id":"bdcd13e3-28a3-41f0-9ace-a20952def3a0","date_create":1683836285,"action":"user_created","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"181.2.69.143"}},
{"id":"0126a45b-6c7d-8900-e12f-3456789gh0i1","date_create":1683836287,"action":"user_login","actor":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"entity":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"context":{"location":{"type":"enterprise","id":"E1701NCCA","name":"Birdland","domain":"birdland"},"ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36","ip_address":"81.2.69.143"}}
],
"response_metadata": {
"next_cursor": "GytjmKHF5hFmty"
}
}
- path: /audit/v1/logs
methods: ["GET"]
request_headers:
authorization: Bearer xoxp-1234567890
accept: application/json
query_params:
limit: "2"
cursor: YXNkZmFzZGZhc2Rm
responses:
Expand All @@ -15,8 +70,11 @@ rules:
{
"entries":[
{"action":"file_downloaded","actor":{"type":"user","user":{"email":"user.mcuser@abcd.co","id":"2f52269c-4f38-4f08-b56d-c2b968681dbd","name":"User McUser","team":"user-team"}},"context":{"ip_address":"81.2.69.144","location":{"domain":"domain.tld","id":"eedd1a7d-1a92-418d-8b01-51a4c809d0fb","name":"The Place","type":"workspace"},"session_id":913888259765,"ua":"com.tinyspeck.chatlyio/23.04.40 (iPhone; iOS 1.4.1; Scale/3.00)"},"date_create":1683836275,"details":{"url_private":"https://example.com/"},"entity":{"file":{"filetype":"image/png","id":"7edc4c42-f925-47af-979a-22c10e1fefed","name":"image.png","title":"image.png"},"type":"file"},"id":"2db28060-1659-4b27-ad55-fdba12e3a7b1"},
{"id":"16f5fb41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1673631531,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}}
]
{"id":"16f5fb41-c67c-4cf5-a5c4-d90cb58dd5f9","date_create":1683836271,"action":"anomaly","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"81.2.69.143"},"details":{"action_timestamp":1673631621862,"location":"England, GB","previous_ip_address":"175.16.199.64","previous_ua":"","reason":["asn","ip_address"]}}
],
"response_metadata": {
"next_cursor": ""
}
}
- path: /audit/v1/logs
methods: ["GET"]
Expand All @@ -25,15 +83,14 @@ rules:
accept: application/json
query_params:
oldest: "{oldest:\\d+}"
latest: "{latest:\\d+}"
limit: "2"
responses:
- status_code: 200
body: |-
{
"entries":[
{"id":"bdcb13e3-28a3-41f0-9ace-a20952def3a0","date_create":1566215192,"action":"user_created","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"181.2.69.143"}},
{"id":"0123a45b-6c7d-8900-e12f-3456789gh0i1","date_create":1521214343,"action":"user_login","actor":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"entity":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"context":{"location":{"type":"enterprise","id":"E1701NCCA","name":"Birdland","domain":"birdland"},"ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36","ip_address":"81.2.69.143"}}
{"id":"bdcb13e3-28a3-41f0-9ace-a20952def3a0","date_create":1683836277,"action":"user_created","actor":{"type":"user","user":{"id":"e65b0f5c","name":"roy","email":"aaron@demo.com"}},"entity":{"type":"user","user":{"id":"asdfasdf","name":"Joe Bob","email":"jbob@example.com","team":"T234SAH2"}},"context":{"location":{"type":"workspace","id":"e65b11aa","name":"Docker","domain":"Docker"},"ua":"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0","ip_address":"181.2.69.143"}},
{"id":"0123a45b-6c7d-8900-e12f-3456789gh0i1","date_create":1683836279,"action":"user_login","actor":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"entity":{"type":"user","user":{"id":"W123AB456","name":"Charlie Parker","email":"bird@slack.com"}},"context":{"location":{"type":"enterprise","id":"E1701NCCA","name":"Birdland","domain":"birdland"},"ua":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36","ip_address":"81.2.69.143"}}
],
"response_metadata": {
"next_cursor": "YXNkZmFzZGZhc2Rm"
Expand Down
5 changes: 5 additions & 0 deletions packages/slack/changelog.yml
View file
@@ -1,4 +1,9 @@
# newer versions go on top
- version: "1.10.1"
changes:
- description: Fix handling of API query param 'oldest'
type: bugfix
link: https://github.com/elastic/integrations/pull/6958
- version: "1.10.0"
changes:
- description: Update package-spec to 2.9.0.
Expand Down
4 changes: 3 additions & 1 deletion packages/slack/data_stream/audit/_dev/test/system/test-default-config.yml
View file
Expand Up @@ -9,5 +9,7 @@ data_stream:
preserve_original_event: true
oauth_token: xoxp-1234567890
limit: 2
initial_interval: 4800h
interval: 10s
assert:
hit_count: 4
hit_count: 10
47 changes: 36 additions & 11 deletions packages/slack/data_stream/audit/agent/stream/httpjson.yml.hbs
View file
Expand Up @@ -19,13 +19,27 @@ request.transforms:
- set:
target: header.Authorization
value: "Bearer {{oauth_token}}"

- set:
target: url.params.cursor
value: '[[.cursor.next_cursor]]'
# An empty value omits this param from being set on the request.
# This should be omitted if it is a new request or a request after pagination is complete.
default: ''

- set:
target: url.params.oldest
value: "[[.cursor.last_timestamp]]"
# If last pagination cycle was finished successully
# we move the oldest pointer forward
# else we continue processing using cursor
# If none of the values are in the cursor it means is a fresh start
# and we pick the default value.
value: |-
[[- if .cursor.pagination_finished -]]
[[- .cursor.next_oldest_date -]]
[[- end -]]
default: '[[(now (parseDuration "-{{initial_interval}}")).Unix]]'
- set:
target: url.params.latest
value: '[[(now).Unix]]'

- set:
target: url.params.limit
value: '[[{{limit}}]]'
Expand All @@ -36,15 +50,26 @@ request.rate_limit.remaining: '0' # hardcoded to 0 since slack doesn't return re
response.split:
target: body.entries
response.pagination:
- set:
target: url.params.cursor
value: '[[.last_response.body.response_metadata.next_cursor]]'
fail_on_template_error: true
- set:
target: url.params.cursor
value: '[[.last_response.body.response_metadata.next_cursor]]'
fail_on_template_error: true
- delete:
target: url.params.oldest

cursor:
last_timestamp:
value: "[[toInt .first_event.date_create]]"
fail_on_template_error: true
pagination_finished:
# Use this flag to identify if an execution was interrupted in the middle
# of a pagination cycle.
# For completed pagination next_cursor is "" and if the response is not paginated
# then the blob response_metadata does not exist
value: '[[and (not (index .last_response.body "response_metadata")) (eq (.last_response.body.response_metadata.Get "next_cursor") "")]]'
next_cursor:
# Use this value to be able to resume from an interrupted pagination cycle.
value: '[[.last_response.body.response_metadata.Get "next_cursor"]]'
next_oldest_date:
# In order to pick the next startDate we keep the first event (newest) date.
value: '[[toInt .first_event.date_create]]'

{{#if tags.length}}
tags:
Expand Down
2 changes: 1 addition & 1 deletion packages/slack/manifest.yml
View file
@@ -1,7 +1,7 @@
format_version: 2.9.0
name: slack
title: "Slack Logs"
version: "1.10.0"
version: "1.10.1"
description: "Slack Logs Integration"
type: integration
categories:
Expand Down

0 comments on commit ab4d3a1

Please sign in to comment.