Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[SplunkPy] Fix An Issue with parse response #28451

Merged
merged 12 commits into from Jul 31, 2023
Merged

[SplunkPy] Fix An Issue with parse response #28451

Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
9716bd7
fix an issue
shmuel44 Jul 24, 2023
abd3839
rn
shmuel44 Jul 24, 2023
41a17ad
Merge branch 'master' into sk_splunk_issue
shmuel44 Jul 25, 2023
1e5757c
revert gitlab changed
shmuel44 Jul 25, 2023
fe58473
RN
shmuel44 Jul 25, 2023
28063e9
merge master
shmuel44 Jul 25, 2023
9efd53d
revert gitlab changed
shmuel44 Jul 25, 2023
e32e341
sdk words
shmuel44 Jul 25, 2023
70a9e08
Merge branch 'master' into sk_splunk_issue
shmuel44 Jul 25, 2023
0964e25
fix
shmuel44 Jul 30, 2023
797f7c0
merge master
shmuel44 Jul 30, 2023
0f34846
UD
shmuel44 Jul 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion Packs/SplunkPy/Integrations/SplunkPy/SplunkPy.py
View file
Open in desktop
Expand Up @@ -2057,7 +2057,7 @@ def parse_batch_of_results(current_batch_of_results, max_results_to_add, app):
if "Error in" in item.message:
raise ValueError(item.message)
demisto.debug(f"Splunk-SDK message: {item.message}")
parsed_batch_results.append(convert_to_str(item.message))
parsed_batch_results.append(item.message)

elif isinstance(item, dict):
if demisto.get(item, 'host'):
Expand Down
321 changes: 187 additions & 134 deletions Packs/SplunkPy/Integrations/SplunkPy/SplunkPy_test.py
View file
Open in desktop
Expand Up @@ -95,58 +95,60 @@
'"a1d57ed2-1fe6-4800-be7a-26cd89bf686d","SESSIONID":"stY46PpweFToT5JX04CZGMeCvP8=","ORGOID":' \
'"G2SY6MR3ATKA232T","AOID":"G2N2TJETBRAAXAAA","MSGID":"a1d57ed2-1fe6-0000-be7a-26cd89bf686d"}'

SAMPLE_RESPONSE = [{
'_bkt': 'notable~668~66D21DF4-F4FD-4886-A986-82E72ADCBFE9',
'_cd': '668:17198',
'_indextime': '1596545116',
'_raw': '1596545116, search_name="Endpoint - Recurring Malware Infection - Rule", count="17", '
'day_count="8", dest="ACME-workstation-012", info_max_time="1596545100.000000000", '
'info_min_time="1595939700.000000000", info_search_time="1596545113.965466000", '
'signature="Trojan.Gen.2"',
'_serial': '50',
'_si': ['ip-172-31-44-193', 'notable'],
'_sourcetype': 'stash',
'_time': '2020-08-04T05:45:16.000-07:00',
'dest': 'ACME-workstation-012',
'dest_asset_id': '028877d3c80cb9d87900eb4f9c9601ea993d9b63',
'dest_asset_tag': ['cardholder', 'pci', 'americas'],
'dest_bunit': 'americas',
'dest_category': ['cardholder', 'pci'],
'dest_city': 'Pleasanton',
'dest_country': 'USA',
'dest_ip': '192.168.3.12',
'dest_is_expected': 'TRUE',
'dest_lat': '37.694452',
'dest_long': '-121.894461',
'dest_nt_host': 'ACME-workstation-012',
'dest_pci_domain': ['trust', 'cardholder'],
'dest_priority': 'medium',
'dest_requires_av': 'TRUE',
'dest_risk_object_type': 'system',
'dest_risk_score': '15680',
'dest_should_timesync': 'TRUE',
'dest_should_update': 'TRUE',
'host': 'ip-172-31-44-193',
'host_risk_object_type': 'system',
'host_risk_score': '0',
'index': 'notable',
'linecount': '1',
'priorities': 'medium',
'priority': 'medium',
'risk_score': '15680',
'rule_description': 'Endpoint - Recurring Malware Infection - Rule',
'rule_name': 'Endpoint - Recurring Malware Infection - Rule',
'rule_title': 'Endpoint - Recurring Malware Infection - Rule',
'security_domain': 'Endpoint - Recurring Malware Infection - Rule',
'severity': 'unknown',
'signature': 'Trojan.Gen.2',
'source': 'Endpoint - Recurring Malware Infection - Rule',
'sourcetype': 'stash',
'splunk_server': 'ip-172-31-44-193',
'urgency': 'low',
'owner': 'unassigned',
'event_id': '66D21DF4-F4FD-4886-A986-82E72ADCBFE9@@notable@@5aa44496ec8e5cf45c78ab230189a4ca',
}]
SAMPLE_RESPONSE = [
results.Message("INFO-TEST", "test message"),
{
'_bkt': 'notable~668~66D21DF4-F4FD-4886-A986-82E72ADCBFE9',
'_cd': '668:17198',
'_indextime': '1596545116',
'_raw': '1596545116, search_name="Endpoint - Recurring Malware Infection - Rule", count="17", '
'day_count="8", dest="ACME-workstation-012", info_max_time="1596545100.000000000", '
'info_min_time="1595939700.000000000", info_search_time="1596545113.965466000", '
'signature="Trojan.Gen.2"',
'_serial': '50',
'_si': ['ip-172-31-44-193', 'notable'],
'_sourcetype': 'stash',
'_time': '2020-08-04T05:45:16.000-07:00',
'dest': 'ACME-workstation-012',
'dest_asset_id': '028877d3c80cb9d87900eb4f9c9601ea993d9b63',
'dest_asset_tag': ['cardholder', 'pci', 'americas'],
'dest_bunit': 'americas',
'dest_category': ['cardholder', 'pci'],
'dest_city': 'Pleasanton',
'dest_country': 'USA',
'dest_ip': '192.168.3.12',
'dest_is_expected': 'TRUE',
'dest_lat': '37.694452',
'dest_long': '-121.894461',
'dest_nt_host': 'ACME-workstation-012',
'dest_pci_domain': ['trust', 'cardholder'],
'dest_priority': 'medium',
'dest_requires_av': 'TRUE',
'dest_risk_object_type': 'system',
'dest_risk_score': '15680',
'dest_should_timesync': 'TRUE',
'dest_should_update': 'TRUE',
'host': 'ip-172-31-44-193',
'host_risk_object_type': 'system',
'host_risk_score': '0',
'index': 'notable',
'linecount': '1',
'priorities': 'medium',
'priority': 'medium',
'risk_score': '15680',
'rule_description': 'Endpoint - Recurring Malware Infection - Rule',
'rule_name': 'Endpoint - Recurring Malware Infection - Rule',
'rule_title': 'Endpoint - Recurring Malware Infection - Rule',
'security_domain': 'Endpoint - Recurring Malware Infection - Rule',
'severity': 'unknown',
'signature': 'Trojan.Gen.2',
'source': 'Endpoint - Recurring Malware Infection - Rule',
'sourcetype': 'stash',
'splunk_server': 'ip-172-31-44-193',
'urgency': 'low',
'owner': 'unassigned',
'event_id': '66D21DF4-F4FD-4886-A986-82E72ADCBFE9@@notable@@5aa44496ec8e5cf45c78ab230189a4ca',
}]

EXPECTED = {
"action": "allowed",
Expand Down Expand Up @@ -927,76 +929,132 @@ def test_get_last_update_in_splunk_time(last_update, demisto_params, splunk_time
splunk.get_last_update_in_splunk_time(last_update)


@pytest.mark.parametrize("notable_data, func_call_kwargs, expected_closure_data",
[
# A Notable with a "Closed" status label
({'status_label': 'Closed', 'event_id': 'id', 'status_end': 'true'},
{'close_incident': True, 'close_end_statuses': False, 'close_extra_labels': []},
{'Type': EntryType.NOTE,
'Contents': {
'dbotIncidentClose': True,
'closeReason': 'Notable event was closed on Splunk with status "Closed".',
},
'ContentsFormat': EntryFormat.JSON,
},
),

# A Notable with a "New" status label (shouldn't close)
({'status_label': 'New', 'event_id': 'id', 'status_end': 'false'},
{'close_incident': True, 'close_end_statuses': False, 'close_extra_labels': []},
None,
),

# A Notable with a custom status label that is on close_extra_labels (should close)
({'status_label': 'Custom', 'event_id': 'id', 'status_end': 'false'},
{'close_incident': True, 'close_end_statuses': False, 'close_extra_labels': ['Custom']},
{'Type': EntryType.NOTE,
'Contents': {
'dbotIncidentClose': True,
'closeReason': 'Notable event was closed on Splunk with status "Custom".',
},
'ContentsFormat': EntryFormat.JSON,
},
),

# A Notable with close_extra_labels that don't include status_label (shouldn't close)
({'status_label': 'Custom', 'event_id': 'id', 'status_end': 'false'},
{'close_incident': True, 'close_end_statuses': False, 'close_extra_labels': ['A', 'B']},
None,
),

# A Notable that has status_end as true with close_end_statuses as true (should close)
({'status_label': 'Custom', 'event_id': 'id', 'status_end': 'true'},
{'close_incident': True, 'close_end_statuses': True, 'close_extra_labels': []},
{'Type': EntryType.NOTE,
'Contents': {
'dbotIncidentClose': True,
'closeReason': 'Notable event was closed on Splunk with status "Custom".',
},
'ContentsFormat': EntryFormat.JSON,
},
),

# A Notable that has status_end as true with close_end_statuses as false (shouldn't close)
({'status_label': 'Custom', 'event_id': 'id', 'status_end': 'true'},
{'close_incident': True, 'close_end_statuses': False, 'close_extra_labels': []},
None,
),

# A Notable that is both on close_extra_labels,
# and has status_end as true with close_end_statuses as true (should close)
({'status_label': 'Custom', 'event_id': 'id', 'status_end': 'true'},
{'close_incident': True, 'close_end_statuses': True, 'close_extra_labels': ['Custom']},
{'Type': EntryType.NOTE,
'Contents': {
'dbotIncidentClose': True,
'closeReason': 'Notable event was closed on Splunk with status "Custom".',
},
'ContentsFormat': EntryFormat.JSON,
},
),
])
def test_get_remote_data_command_close_incident(mocker, notable_data: dict,
@pytest.mark.parametrize(
"notable_data, func_call_kwargs, expected_closure_data",
[
# A Notable with a "Closed" status label
(
[
results.Message("INFO-TEST", "test message"),
{"status_label": "Closed", "event_id": "id", "status_end": "true"},
],
{
"close_incident": True,
"close_end_statuses": False,
"close_extra_labels": [],
},
{
"Type": EntryType.NOTE,
"Contents": {
"dbotIncidentClose": True,
"closeReason": 'Notable event was closed on Splunk with status "Closed".',
},
"ContentsFormat": EntryFormat.JSON,
},
),
# A Notable with a "New" status label (shouldn't close)
(
[
results.Message("INFO-TEST", "test message"),
{"status_label": "New", "event_id": "id", "status_end": "false"},
],
{
"close_incident": True,
"close_end_statuses": False,
"close_extra_labels": [],
},
None,
),
# A Notable with a custom status label that is on close_extra_labels (should close)
(
[
results.Message("INFO-TEST", "test message"),
{"status_label": "Custom", "event_id": "id", "status_end": "false"},
],
{
"close_incident": True,
"close_end_statuses": False,
"close_extra_labels": ["Custom"],
},
{
"Type": EntryType.NOTE,
"Contents": {
"dbotIncidentClose": True,
"closeReason": 'Notable event was closed on Splunk with status "Custom".',
},
"ContentsFormat": EntryFormat.JSON,
},
),
# A Notable with close_extra_labels that don't include status_label (shouldn't close)
(
[
results.Message("INFO-TEST", "test message"),
{"status_label": "Custom", "event_id": "id", "status_end": "false"},
],
{
"close_incident": True,
"close_end_statuses": False,
"close_extra_labels": ["A", "B"],
},
None,
),
# A Notable that has status_end as true with close_end_statuses as true (should close)
(
[
results.Message("INFO-TEST", "test message"),
{"status_label": "Custom", "event_id": "id", "status_end": "true"},
],
{
"close_incident": True,
"close_end_statuses": True,
"close_extra_labels": [],
},
{
"Type": EntryType.NOTE,
"Contents": {
"dbotIncidentClose": True,
"closeReason": 'Notable event was closed on Splunk with status "Custom".',
},
"ContentsFormat": EntryFormat.JSON,
},
),
# A Notable that has status_end as true with close_end_statuses as false (shouldn't close)
(
[
results.Message("INFO-TEST", "test message"),
{"status_label": "Custom", "event_id": "id", "status_end": "true"},
],
{
"close_incident": True,
"close_end_statuses": False,
"close_extra_labels": [],
},
None,
),
# A Notable that is both on close_extra_labels,
# and has status_end as true with close_end_statuses as true (should close)
(
[
results.Message("INFO-TEST", "test message"),
{"status_label": "Custom", "event_id": "id", "status_end": "true"},
],
{
"close_incident": True,
"close_end_statuses": True,
"close_extra_labels": ["Custom"],
},
{
"Type": EntryType.NOTE,
"Contents": {
"dbotIncidentClose": True,
"closeReason": 'Notable event was closed on Splunk with status "Custom".',
},
"ContentsFormat": EntryFormat.JSON,
},
),
],
)
def test_get_remote_data_command_close_incident(mocker, notable_data: list[results.Message | dict],
func_call_kwargs: dict, expected_closure_data: dict):
class Jobs:
def oneshot(self, _, output_mode: str):
Expand All @@ -1011,13 +1069,13 @@ def __init__(self):
mocker.patch.object(demisto, 'params', return_value={'timezone': '0'})
mocker.patch.object(demisto, 'debug')
mocker.patch.object(demisto, 'info')
mocker.patch('SplunkPy.results.JSONResultsReader', return_value=[notable_data])
mocker.patch('SplunkPy.results.JSONResultsReader', return_value=notable_data)
mocker.patch.object(demisto, 'results')
service = Service()
splunk.get_remote_data_command(service, args, mapper=splunk.UserMappingObject(service, False), **func_call_kwargs)
results = demisto.results.call_args[0][0]

expected_results = [notable_data]
expected_results = [notable_data[1]]

if expected_closure_data:
expected_results.append(expected_closure_data)
Expand All @@ -1039,15 +1097,10 @@ def test_get_remote_data_command_with_message(mocker):
Returns:
None
"""

class Message:
def __init__(self, message: str):
self.message = message

class Jobs:
def oneshot(self, _, output_mode: str):
assert output_mode == splunk.OUTPUT_MODE_JSON
return Message("test message")
return results.Message("INFO-test", "test message")

class Service:
def __init__(self):
Expand All @@ -1064,7 +1117,7 @@ def __init__(self):
mocker.patch.object(demisto, "params", return_value={"timezone": "0"})
mocker.patch.object(demisto, "info")
mocker.patch(
"SplunkPy.results.JSONResultsReader", return_value=[Message("test message")]
"SplunkPy.results.JSONResultsReader", return_value=[results.Message("INFO-test", "test message")]
)
mocker.patch("SplunkPy.isinstance", return_value=True)

Expand Down Expand Up @@ -1586,15 +1639,15 @@ def test_labels_with_non_str_values(mocker):
"bool_val": False,
"float_val": 100.0
}
mocked_response = SAMPLE_RESPONSE[0].copy()
mocked_response['_raw'] = json.dumps(raw)
mocked_response: list[results.Message | dict] = SAMPLE_RESPONSE.copy()
mocked_response[1]['_raw'] = json.dumps(raw)
mock_last_run = {'time': '2018-10-24T14:13:20'}
mock_params = {'fetchQuery': "something", "parseNotableEventsRaw": True}
mocker.patch.object(demisto, 'incidents')
mocker.patch.object(demisto, 'setLastRun')
mocker.patch('demistomock.getLastRun', return_value=mock_last_run)
mocker.patch('demistomock.params', return_value=mock_params)
mocker.patch('splunklib.results.JSONResultsReader', return_value=[mocked_response])
mocker.patch('splunklib.results.JSONResultsReader', return_value=mocked_response)

# run
service = mocker.patch('splunklib.client.connect', return_value=None)
Expand Down
6 changes: 6 additions & 0 deletions Packs/SplunkPy/ReleaseNotes/3_1_1.md
View file
Open in desktop
@@ -0,0 +1,6 @@

#### Integrations

##### SplunkPy

Fixed an issue where the ***splunk-search*** command failed when splunk-sdk returned a message object.