Skip to content

Commit

Permalink
31078 misp (#32237)
Browse files Browse the repository at this point in the history 
* adding debug to indicators

* adding the unify yml

* fixed custom version

* Update README.md (#23810)

Edit the file to remove duplication of command names in the right pane.

* Modeling rules fixes (#24259)

* save

* save no exit_code

* save not fail on test-modeling-rules

* remove ciscoasa changes

* Update Docker Image To demisto/chromium  (#24291)

* Updated Metadata Of Pack ExpanseV2

* Added release notes to pack ExpanseV2

* Packs/ExpanseV2/Scripts/ExpanseGenerateIssueMapWidgetScript/ExpanseGenerateIssueMapWidgetScript.yml Docker image update

* Update Docker Image To demisto/chromium  (#24291)

* Updated Metadata Of Pack ExpanseV2

* Added release notes to pack ExpanseV2

* Packs/ExpanseV2/Scripts/ExpanseGenerateIssueMapWidgetScript/ExpanseGenerateIssueMapWidgetScript.yml Docker image update

* release note

* Delete .gitlab/ci/on-push.yml

* Delete Packs/Auditd/ModelingRules/Auditd_1_3/Auditd_1_3_testdata.json

* try

* fix

* fix

* adding limit to fetch indicators

* adding to docs

* fixing limit

* fix val

* custom integration

* fix cr

* testing

* fix pre-commit

* Update Packs/FeedMISP/ReleaseNotes/1_0_31.md

Co-authored-by: ShirleyDenkberg <62508050+ShirleyDenkberg@users.noreply.github.com>

* changes fix

* limit and page size together

* fix

* fix fetch

* fix rl

* adding unit tests

* fix doc string

* fix

* not to push

* cr notes

* cr notes ream me

* cr notes - fix docstring test

* cr notes - fix command, yml, timestamp

* cr notes - fix duplicates

* pre commit

* pre commit fix

* cr notes

* fix unit test

* Apply suggestions from code review

Co-authored-by: EyalPintzov <91007713+eyalpalo@users.noreply.github.com>

---------

Co-authored-by: ShirleyDenkberg <62508050+ShirleyDenkberg@users.noreply.github.com>
Co-authored-by: Darya Koval <72339940+daryakoval@users.noreply.github.com>
Co-authored-by: content-bot <55035720+content-bot@users.noreply.github.com>
Co-authored-by: EyalPintzov <91007713+eyalpalo@users.noreply.github.com>
  • Loading branch information
@maimorag @ShirleyDenkberg
@daryakoval @content-bot @eyalpalo
5 people committed Jan 31, 2024
1 parent 8577b20 commit 89f16e9
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 269 deletions.
115 changes: 32 additions & 83 deletions Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.py
View file
Expand Up @@ -166,7 +166,6 @@ def build_indicators_iterator(attributes: Dict[str, Any], url: Optional[str]) ->
except KeyError as err:
demisto.debug(str(err))
raise KeyError(f'Could not parse returned data as attributes list. \nError massage: {err}')
demisto.debug(f' Number of indicators: {len(indicators_iterator)}')
return indicators_iterator


Expand Down Expand Up @@ -211,7 +210,7 @@ def handle_file_type_fields(raw_type: str, indicator_obj: Dict[str, Any]) -> Non
indicator_obj['fields'][raw_type.upper()] = hash_value


def build_params_dict(tags: List[str], attribute_type: List[str]) -> Dict[str, Any]:
def build_params_dict(tags: List[str], attribute_type: List[str], limit: int, page: int) -> Dict[str, Any]:
"""
Creates a dictionary in the format required by MISP to be used as a query.
Args:
Expand All @@ -227,6 +226,8 @@ def build_params_dict(tags: List[str], attribute_type: List[str]) -> Dict[str, A
'tags': {
'OR': tags if tags else [],
},
'limit': limit,
'page': page
}
return params

Expand Down Expand Up @@ -307,82 +308,15 @@ def build_indicator(value_: str, type_: str, raw_data: Dict[str, Any], reputatio
return indicator_obj


def update_indicators_iterator(indicators_iterator: List[Dict[str, Any]],
params_dict: Dict[str, Any],
is_fetch: bool) -> Optional[List[Dict[str, Any]]]:
"""
sorts the indicators by their timestamp and returns a list of only new indicators received from MISP
Args:
params_dict: user's params sent to misp
indicators_iterator: list of indicators
is_fetch: flag for wether funciton was called for fetching command or a get
Returns: Sorted list of new indicators
"""
last_run = demisto.getLastRun()
demisto.debug(f"last_run: {last_run}")
indicators_iterator.sort(key=lambda indicator: indicator['value']['timestamp'])

if last_run is None:
return indicators_iterator
if params_dict != last_run.get('params'):
if is_fetch:
demisto.setLastRun(None)
return indicators_iterator

last_timestamp = int(last_run.get('timestamp'))

for index in range(len(indicators_iterator)):
if int(indicators_iterator[index]['value']['timestamp']) > last_timestamp:
return indicators_iterator[index:]
return []


def search_query_indicators_pagination(client: Client, params_dict: Dict[str, Any]) -> Dict[str, Any]:
params_dict['page'] = 1
response: Dict[str, Dict[str, List]] = {'response': {'Attribute': []}}
search_query_per_page = client.search_query(params_dict).get('response', {}).get('Attribute')
while len(search_query_per_page):
demisto.debug(f'search_query_per_page: {params_dict["page"]} number of indicators: {len(search_query_per_page)}')
response['response']['Attribute'].extend(search_query_per_page)
params_dict['page'] += 1
search_query_per_page = client.search_query(params_dict).get('response', {}).get('Attribute')
return response


def fetch_indicators(client: Client,
tags: List[str],
def build_indicators(response: Dict[str, Any],
attribute_type: List[str],
query: Optional[str],
tlp_color: Optional[str],
url: Optional[str],
reputation: Optional[str],
feed_tags: Optional[List],
limit: int = -1,
is_fetch: bool = True) -> List[Dict]:
params_dict = clean_user_query(query) if query else build_params_dict(tags, attribute_type)
if limit and limit not in params_dict:
params_dict['limit'] = limit
response = search_query_indicators_pagination(client, params_dict) if is_fetch else client.search_query(params_dict)
if error_message := response.get('Error'):
raise DemistoException(error_message)
feed_tags: Optional[List]) -> List[Dict]:
indicators_iterator = build_indicators_iterator(response, url)
added_indicators_iterator = update_indicators_iterator(indicators_iterator, params_dict, is_fetch)
indicators = []

if not added_indicators_iterator:
return []

if limit > 0:
added_indicators_iterator = added_indicators_iterator[:limit]

if is_fetch:
# fetching command, need to update last run dict
demisto.setLastRun({
'params': params_dict,
'timestamp': added_indicators_iterator[len(added_indicators_iterator) - 1]['value']['timestamp']
})

for indicator in added_indicators_iterator:
for indicator in indicators_iterator:
value_ = indicator['value']['value']
type_ = indicator['type']
raw_type = indicator.pop('raw_type')
Expand Down Expand Up @@ -522,7 +456,7 @@ def test_module(client: Client) -> str:


def get_attributes_command(client: Client, args: Dict[str, str], params: Dict[str, str]) -> CommandResults:
"""Wrapper for retrieving indicators from the feed to the war-room.
""" Wrapper for fetching indicators from the feed to the war-room.
Args:
client: Client object with request
args: demisto.args()
Expand All @@ -537,8 +471,13 @@ def get_attributes_command(client: Client, args: Dict[str, str], params: Dict[st
feed_tags = argToList(params.get("feedTags", []))
query = args.get('query', None)
attribute_type = argToList(args.get('attribute_type', ''))
indicators = fetch_indicators(client, tags, attribute_type,
query, tlp_color, params.get('url'), reputation, feed_tags, limit, False)
page = arg_to_number(args.get('page')) or 1
params_dict = clean_user_query(query) if query else build_params_dict(tags=tags, attribute_type=attribute_type, limit=limit,
page=page)
response = client.search_query(params_dict)
if error_message := response.get('Error'):
raise DemistoException(error_message)
indicators = build_indicators(response, attribute_type, tlp_color, params.get('url'), reputation, feed_tags)
hr_indicators = []
for indicator in indicators:
hr_indicators.append({
Expand All @@ -558,23 +497,34 @@ def get_attributes_command(client: Client, args: Dict[str, str], params: Dict[st
)


def fetch_attributes_command(client: Client, params: Dict[str, str]) -> List[Dict]:
def fetch_attributes_command(client: Client, params: Dict[str, str]):
"""
Wrapper for fetching indicators from the feed to the Indicators tab.
Fetching indicators from the feed to the Indicators tab.
Args:
client: Client object with request
params: demisto.params()
Returns: List of indicators.
"""
tlp_color = params.get('tlp_color')
reputation = params.get('feedReputation')
tags = argToList(params.get('attribute_tags', ''))
feed_tags = argToList(params.get("feedTags", []))
attribute_types = argToList(params.get('attribute_types', ''))
query = params.get('query', None)
indicators = fetch_indicators(client, tags, attribute_types, query, tlp_color,
params.get('url'), reputation, feed_tags)
return indicators
params_dict = clean_user_query(query) if query else build_params_dict(tags=tags, attribute_type=attribute_types, limit=2000,
page=1)
search_query_per_page = client.search_query(params_dict)
while len(search_query_per_page.get("response", {}).get("Attribute", [])):
demisto.debug(f'search_query_per_page number of attributes:\
{len(search_query_per_page.get("response", {}).get("Attribute", []))}\
page: {params_dict["page"]}')
indicators = build_indicators(search_query_per_page, attribute_types, tlp_color, params.get('url'), reputation, feed_tags)
demisto.createIndicators(indicators)
params_dict['page'] += 1
search_query_per_page = client.search_query(params_dict)
if error_message := search_query_per_page.get('Error'):
raise DemistoException(f"Error in API call - check the input parameters and the API Key. Error: {error_message}")


def main():
Expand All @@ -601,9 +551,8 @@ def main():
elif command == 'misp-feed-get-indicators':
return_results(get_attributes_command(client, args, params))
elif command == 'fetch-indicators':
indicators = fetch_attributes_command(client, params)
for iter_ in batch(indicators, batch_size=2000):
demisto.createIndicators(iter_)
fetch_attributes_command(client, params)

else:
raise NotImplementedError(f'Command {command} is not implemented.')

Expand Down
5 changes: 4 additions & 1 deletion Packs/FeedMISP/Integrations/FeedMISP/FeedMISP.yml
View file
Expand Up @@ -127,6 +127,9 @@ script:
- name: limit
defaultValue: '10'
description: The maximum number of results to return.
- name: page
defaultValue: '1'
description: The page number of the results to retrieve.
- name: tags
description: Attributes having one of the tags, or being an attribute of an event having one of the tags, will be returned. You can enter a comma-separated list of tags, for example <tag1>,<tag2>,<tag3>. The list of MISP tags can be found in your MISP instance under 'Event Actions'>'List Tags'.
- name: attribute_type
Expand All @@ -139,7 +142,7 @@ script:
script: '-'
type: python
subtype: python3
dockerimage: demisto/python3:3.10.13.83255
dockerimage: demisto/python3:3.10.13.86272
fromversion: 5.5.0
tests:
- MISPfeed Test
Expand Down

0 comments on commit 89f16e9

Please sign in to comment.