DBotMLFetchData.py

import demistomock as demisto  # noqa: F401
from CommonServerPython import *  # noqa: F401
import uuid
from itertools import combinations

import dateutil

from CommonServerUserPython import *
import json
import pickle
import re
from nltk.tokenize import word_tokenize, sent_tokenize
import string
from bs4 import BeautifulSoup
from collections import Counter
import pandas as pd
import signal
import zlib
from base64 import b64encode
from nltk import ngrams
from datetime import datetime
import tldextract
from email.utils import parseaddr
import onnx
import onnxruntime
import numpy as np
import os
import io

f = io.StringIO()
stderr = sys.stderr
sys.stderr = f
from transformers import DistilBertTokenizer  # noqa: E402

sys.stderr = sys.__stderr__

EXECUTION_JSON_FIELD = 'last_execution'
VERSION_JSON_FIELD = 'script_version'

MAX_ALLOWED_EXCEPTIONS = 20

NO_FETCH_EXTRACT = tldextract.TLDExtract(suffix_list_urls=None, cache_dir=False)
NON_POSITIVE_VALIDATION_VALUES = set(['none', 'fail', 'softfail'])

VALIDATION_OTHER = 'other'

VALIDATION_NA = 'na'

CHARACTERS_TO_COUNT = list(string.printable) + ['$', '€', '£', '¥', '₪', '₽']

MIN_TEXT_LENGTH = 20

MODIFIED_QUERY_TIMEFORMAT = '%Y-%m-%d %H:%M:%S'
EMAIL_BODY_FIELD = 'emailbody'
EMAIL_SUBJECT_FIELD = 'emailsubject'
EMAIL_HTML_FIELD = 'emailbodyhtml'
EMAIL_HEADERS_FIELD = 'emailheaders'
EMAIL_ATTACHMENT_FIELD = 'attachment'

GLOVE_50_PATH = '/ml/glove_50_top_20k.p'
GLOVE_100_PATH = '/ml/glove_100_top_20k.p'
FASTTEXT_PATH = '/ml/fasttext_top_20k.p'
DOMAIN_TO_RANK_PATH = '/ml/domain_to_rank.p'
WORD_TO_NGRAM_PATH = '/ml/word_to_ngram.p'
WORD_TO_REGEX_PATH = '/ml/word_to_regex.p'

EMBEDDING_DICT_GLOVE_50 = None
EMBEDDING_DICT_GLOVE_100 = None
EMBEDDING_DICT_FASTTEXT = None
DOMAIN_TO_RANK = None
WORD_TO_REGEX = None
WORD_TO_NGRAMS = None

FETCH_DATA_VERSION = '4.0'
LAST_EXECUTION_LIST_NAME = 'FETCH_DATA_ML_LAST_EXECUTION'
MAX_INCIDENTS_TO_FETCH_PERIODIC_EXECUTION = 400
MAX_INCIDENTS_TO_FETCH_FIRST_EXECUTION = 3000
FROM_DATA_FIRST_EXECUTION = FROM_DATA_PERIODIC_EXECUTION = '30 days ago'
DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.%f'
FIRST_EXECUTION_ARGUMENTS = {'limit': MAX_INCIDENTS_TO_FETCH_FIRST_EXECUTION, 'fromDate': FROM_DATA_FIRST_EXECUTION}

IMG_FORMATS = ['.jpeg', '.gif', '.bmp', '.png', '.jfif', '.tiff', '.eps', '.indd', '.jpg']

RECEIVED_SERVER_REGEX = r'(?<=from )[a-zA-Z0-9_.+-]+'
ENVELOP_FROM_REGEX = r'(?<=envelope-from )<?[a-zA-Z0-9@_\.\+\-]+>?'

IP_DOMAIN_TOKEN = 'IP_DOMAIN'

'''
Define time out functionality
'''


class TimeoutException(Exception):  # Custom exception class
    pass


def timeout_handler(signum, frame):  # Custom signal handler
    if signum or frame:
        pass
    raise TimeoutException


# Change the behavior of SIGALRM
signal.signal(signal.SIGALRM, timeout_handler)


class ShortTextException(Exception):
    pass


'''
Define heuristics for finding label field
'''
LABEL_FIELDS_BLACKLIST = {EMAIL_BODY_FIELD, EMAIL_SUBJECT_FIELD, EMAIL_HTML_FIELD, 'emailbodyhtml', 'CustomFields',
                          'ShardID', 'account', 'activated', 'attachment', 'autime', 'canvases', 'category',
                          'closeNotes', 'closed', 'closingUserId', 'created', 'criticalassets', 'dbotCreatedBy',
                          'dbotMirrorDirection', 'dbotMirrorId', 'details', 'detectionsla', 'droppedCount',
                          'dbotMirrorInstance', 'dbotMirrorLastSync', 'dueDate', 'emailbody', 'emailcc', 'emailfrom',
                          'emailhtml', 'emailmessageid', 'emailto', 'emailtocount', 'emailheaders', 'emailsubject',
                          'hasRole', 'id', 'investigationId', 'isPlayground', 'labels', 'lastJobRunTime', 'lastOpen',
                          'linkedCount', 'linkedIncidents', 'modified', 'name', 'notifyTime', 'occurred',
                          'openDuration', 'owner', 'parent', 'phase', 'playbookId', 'previousRoles', 'rawCategory',
                          'rawCloseReason', 'rawJSON', 'rawName', 'rawPhase', 'rawType', 'reason', 'remediationsla',
                          'reminder', 'roles', 'runStatus', 'severity', 'sla', 'sortValues', 'sourceBrand',
                          'sourceInstance', 'status', 'timetoassignment', 'type', 'urlsslverification', 'version',
                          "index", 'allRead', 'allReadWrite', 'dbotCurrentDirtyFields', 'dbotDirtyFields',
                          'dbotMirrorTags', 'feedBased', 'previousAllRead', 'previousAllReadWrite',
                          'dbottextsuggestionhighlighted'}
LABEL_VALUES_KEYWORDS = ['spam', 'malicious', 'legit', 'false', 'positive', 'phishing', 'fraud', 'internal', 'test',
                         'fp', 'tp', 'resolve', 'credentials', 'spear', 'malware', 'whaling', 'catphishing',
                         'catfishing', 'social', 'sextortion', 'blackmail', 'spyware', 'adware']
LABEL_FIELD_KEYWORDS = ['classifi', 'type', 'resolution', 'reason', 'categor', 'disposition', 'severity', 'malicious',
                        'tag', 'close', 'phish', 'phishing']
LABEL_FIELD_BLACKLIST_KEYWORDS = ['attach', 'recipient']

'''
Define html tags to count
'''
HTML_TAGS = ["a", "abbr", "acronym", "address", "area", "b", "base", "bdo", "big", "blockquote", "body", "br", "button",
             "caption", "cite", "code", "col", "colgroup", "dd", "del", "dfn", "div", "dl", "DOCTYPE", "dt", "em",
             "fieldset", "figcaption", "figure", "footer", "form", "h1", "h2", "h3", "h4", "h5", "h6", "head", "html",
             "hr", "i", "img", "input", "ins", "invalidtag", "kbd", "label", "legend", "li", "link", "map", "meta",
             "noscript", "object", "ol", "optgroup", "option", "p", "param", "pre", "q", "samp", "script", "section",
             "select", "small", "span", "strong", "style", "sub", "sup", "table", "tbody", "td", "textarea", "tfoot",
             "th", "thead", "title", "tr", "tt", "ul", "var"]
HTML_TAGS = set(HTML_TAGS)

'''
Define known shortened and drive domains
'''

DRIVE_URL_KEYWORDS = ['drive', 'transfer', 'formplus', 'dropbox', 'sendspace', 'onedrive', 'box', 'pcloud', 'icloud',
                      'mega', 'spideroak', 'sharepoint']
SHORTENED_DOMAINS = {"adf.ly", "t.co", "goo.gl", "adbooth.net", "adfoc.us", "bc.vc", "bit.ly", "j.gs", "seomafia.net",
                     "adlock.in", "adbooth.com", "cutt.us", "is.gd", "cf.ly", "ity.im", "tiny.cc", "adfa.st",
                     "budurl.com", "soo.gd", "prettylinkpro.com", "shrinkonce.com", "ad7.biz", "2tag.nl", "1o2.ir",
                     "hotshorturl.com", "onelink.ir", "dai3.net", "9en.us", "kaaf.com", "rlu.ru", "awe.sm", "4ks.net",
                     "s2r.co", "4u2bn.com", "multiurl.com", "tab.bz", "dstats.net", "iiiii.in", "nicbit.com",
                     "l1nks.org", "at5.us", "bizz.cc", "fur.ly", "clicky.me", "magiclinker.com", "miniurl.com",
                     "bit.do", "adurl.biz", "omani.ac", "1y.lt", "1click.im", "1dl.us", "4zip.in", "ad4.us", "adfro.gs",
                     "adnld.com", "adshor.tk", "adspl.us", "adzip.us", "articleshrine.com", "asso.in", "b2s.me",
                     "bih.cc", "biturl.net", "buraga.org", "cc.cr", "cf6.co", "dollarfalls.info", "domainonair.com",
                     "gooplu.com", "hide4.me", "ik.my", "ilikear.ch", "infovak.com", "itz.bz", "jetzt-hier-klicken.de",
                     "kly.so", "lst.bz", "mrte.ch", "multiurlscript.com", "nowlinks.net", "nsyed.com", "ooze.us",
                     "ozn.st", "scriptzon.com", "short2.in", "shortxlink.com", "shr.tn", "shrt.in", "sitereview.me",
                     "sk.gy", "snpurl.biz", "socialcampaign.com", "swyze.com", "theminiurl.com", "tinylord.com",
                     "tinyurl.ms", "tip.pe", "ty.by"}

# ONNX BERT
os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'
ONNX_MODEL = None
ORT_SESSION = None
TOKENIZER = None


def hash_value(simple_value):
    if not isinstance(simple_value, str):
        simple_value = str(simple_value)
    if simple_value.lower() in ["none", "null"]:
        return None
    return hash_djb2(simple_value)


def find_label_fields_candidates(incidents_df):
    candidates = [col for col in list(incidents_df) if
                  sum(isinstance(x, str) or isinstance(x, bool) for x in incidents_df[col]) > 0.3 * len(incidents_df)]
    candidates = [col for col in candidates if col not in LABEL_FIELDS_BLACKLIST]
    candidates = [col for col in candidates if not any(w in col.lower() for w in LABEL_FIELD_BLACKLIST_KEYWORDS)]
    candidate_to_values = {col: incidents_df[col].unique() for col in candidates}
    candidate_to_values = {col: values for col, values in candidate_to_values.items() if
                           sum(not (isinstance(v, str) or isinstance(v, bool)) for v in values) <= 1}

    # filter columns by unique values count
    candidate_to_values = {col: values for col, values in candidate_to_values.items() if 0 < len(values) < 15}
    candidate_to_values = {col: [v.lower() if isinstance(v, str) else v for v in values]
                           for col, values in candidate_to_values.items()}
    candidate_to_score = {col: 0 for col in candidate_to_values}
    for col, values in candidate_to_values.items():
        for v in values:
            if any(isinstance(v, str) and w in v for w in LABEL_VALUES_KEYWORDS):
                candidate_to_score[col] += 1
        if any(w in col.lower() for w in LABEL_FIELD_KEYWORDS):
            candidate_to_score[col] += 1
    return [k for k, v in sorted(candidate_to_score.items(), key=lambda item: item[1], reverse=True)]


def get_text_from_html(html):
    soup = BeautifulSoup(html)
    # kill all script and style elements
    for script in soup(["script", "style"]):
        script.extract()  # rip it out
    # get text
    text = soup.get_text()
    # break into lines and remove leading and trailing space on each
    lines = (line.strip() for line in text.splitlines())
    # break multi-headlines into a line each
    chunks = (phrase.strip() for line in lines for phrase in line.split("  "))
    # drop blank lines
    text = '\n'.join(chunk for chunk in chunks if chunk)
    return text


def get_ngrams_features(text, ngrams_counter):
    global WORD_TO_NGRAMS, WORD_TO_REGEX
    ngrams_features = {}
    for w, regex in WORD_TO_REGEX.items():  # type: ignore
        count = len(re.findall(regex, text))
        if count > 0:
            ngrams_features[w] = count
    for w, ngram in WORD_TO_NGRAMS.items():  # type: ignore
        count = ngrams_counter[ngram] if ngram in ngrams_counter else 0
        if count > 0:
            ngrams_features[w] = count
    return ngrams_features


def get_lexical_features(email_subject, email_body, email_body_word_tokenized, email_subject_word_tokenized):
    words = [w for w in email_body_word_tokenized if any(c.isalnum() for c in w)]
    num_of_words = len(words)
    avg_word_length = 0 if len(words) == 0 else float(sum([len(w) for w in words])) / len(words)
    sentences = sent_tokenize(email_body)
    num_of_sentences = len(sentences)
    avg_sentence_length = 0 if len(sentences) == 0 else float(sum([len(s) for s in sentences])) / len(sentences)
    text_length = len(email_body)
    subject_length = len(email_subject)
    number_of_lines = email_body.count('\n') + 1
    word_subject = [w for w in email_subject_word_tokenized if any(c.isalnum() for c in w)]
    num_of_words_subject = len(word_subject)
    ending_dots = len([s for s in sentences if s.strip().endswith('.')])
    ending_explanation = len([s for s in sentences if s.strip().endswith('!')])
    ending_question = len([s for s in sentences if s.strip().endswith('?')])

    return {'num_of_words': num_of_words,
            'avg_word_length': avg_word_length,
            'num_of_sentences': num_of_sentences,
            'avg_sentence_length': avg_sentence_length,
            'avg_number_of_words_in_sentence': float(num_of_words) / num_of_sentences if num_of_sentences > 0 else 0,
            'text_length': text_length,
            'num_of_words_subject': num_of_words_subject,
            'subject_length': subject_length,
            'ending_dots': ending_dots,
            'ending_explanation': ending_explanation,
            'ending_question': ending_question,
            'number_of_lines': number_of_lines
            }


def get_characters_features(text):
    characters_count = {}
    for c in CHARACTERS_TO_COUNT:
        if c in ['[', ']', '<']:
            continue
        count = text.count(c)
        if count > 0:
            characters_count[c] = count
    return characters_count


def get_url_features(email_body, email_html, soup):
    url_regex = r'(?:(?:https?|ftp|hxxps?):\/\/|www\[?\.\]?|ftp\[?\.\]?)(?:[-\w\d]+\[?\.\]?)+[-\w\d]+(?::\d+)?' \
                r'(?:(?:\/|\?)[-\w\d+&@#\/%=~_$?!\-:,.\(\);]*[\w\d+&@#\/%=~_$\(\);])?'
    embedded_urls = []
    for a in soup.findAll('a'):
        if a.has_attr('href'):
            if a['href'] not in a.get_text():
                embedded_urls.append(a['href'])
    plain_urls = re.findall(url_regex, email_body)
    all_urls = plain_urls + embedded_urls
    all_urls_lengths = [len(u) for u in all_urls]
    average_url_length = 0 if len(all_urls) == 0 else sum(all_urls_lengths) / len(all_urls)
    min_url_length = 0 if len(all_urls_lengths) == 0 else min(all_urls_lengths)
    max_url_length = 0 if len(all_urls_lengths) == 0 else max(all_urls_lengths)
    shortened_urls_count = len([u for u in all_urls if any(shortened_u in u for shortened_u in SHORTENED_DOMAINS)])
    drive_count = len([u for u in all_urls if any(drive in u for drive in DRIVE_URL_KEYWORDS)])
    return {
        'http_urls_count': sum(url.startswith('http') and not url.startswith('https') for url in plain_urls),
        'https_urls_count': sum(url.startswith('https') for url in plain_urls),
        'embedded_urls_count': len(embedded_urls),
        'all_urls_count': len(all_urls),
        'average_url_length': average_url_length,
        'min_url_length': min_url_length,
        'max_url_length': max_url_length,
        'shortened_urls_count': shortened_urls_count,
        'drive_count': drive_count
    }


def get_html_features(soup):
    global HTML_TAGS
    html_counter = Counter([tag.name for tag in soup.find_all()])
    for t in HTML_TAGS:
        if t in html_counter:
            html_counter[t] = html_counter[t]
    html_counter = {k: v for k, v in html_counter.items() if k in HTML_TAGS}
    return html_counter


def load_external_resources():
    global EMBEDDING_DICT_GLOVE_50, EMBEDDING_DICT_GLOVE_50, EMBEDDING_DICT_GLOVE_100, EMBEDDING_DICT_FASTTEXT, \
        DOMAIN_TO_RANK, DOMAIN_TO_RANK_PATH, WORD_TO_NGRAMS, WORD_TO_REGEX, ONNX_MODEL, ORT_SESSION, TOKENIZER
    with open(GLOVE_50_PATH, 'rb') as file:
        EMBEDDING_DICT_GLOVE_50 = pickle.load(file)
    with open(GLOVE_100_PATH, 'rb') as file:
        EMBEDDING_DICT_GLOVE_100 = pickle.load(file)
    with open(FASTTEXT_PATH, 'rb') as file:
        EMBEDDING_DICT_FASTTEXT = pickle.load(file)
    with open(DOMAIN_TO_RANK_PATH, 'rb') as file:
        DOMAIN_TO_RANK = pickle.load(file)
    with open(WORD_TO_NGRAM_PATH, 'rb') as file:
        WORD_TO_NGRAMS = pickle.load(file)
    with open(WORD_TO_REGEX_PATH, 'rb') as file:
        WORD_TO_REGEX = pickle.load(file)
    ONNX_MODEL = onnx.load("/ml/distilbert-base-uncased.onnx")
    onnx.checker.check_model(ONNX_MODEL)
    ORT_SESSION = onnxruntime.InferenceSession("/ml/distilbert-base-uncased.onnx")
    TOKENIZER = DistilBertTokenizer.from_pretrained('/ml/distilbert-base-uncased_tokenizer')


def get_avg_embedding_vector_for_text(tokenized_text, embedding_dict, size, prefix):
    vectors = [embedding_dict[w] for w in tokenized_text if w in embedding_dict]
    if len(vectors) == 0:
        mean_vector = np.zeros(size)
    else:
        mean_vector = np.mean(vectors, axis=0)  # type: ignore
    res = {'{}_{}'.format(prefix, str(i)): mean_vector[i].item() for i in range(len(mean_vector))}
    return res


def get_embedding_features(tokenized_text):
    res_glove_50 = get_avg_embedding_vector_for_text(tokenized_text, EMBEDDING_DICT_GLOVE_50, 50, 'glove50')
    res_glove_100 = get_avg_embedding_vector_for_text(tokenized_text, EMBEDDING_DICT_GLOVE_100, 100, 'glove100')
    res_fasttext = get_avg_embedding_vector_for_text(tokenized_text, EMBEDDING_DICT_FASTTEXT, 300, 'fasttext')
    return {**res_glove_50, **res_glove_100, **res_fasttext}


def get_header_value(email_headers, header_name, index=0, ignore_case=False):
    if ignore_case:
        header_name = header_name.lower()
        headers_with_name = [header_dict for header_dict in email_headers if
                             'headername' in header_dict and header_dict['headername'].lower() == header_name]
    else:
        headers_with_name = [header_dict for header_dict in email_headers if
                             'headername' in header_dict and header_dict['headername'] == header_name]
    if len(headers_with_name) == 0:
        return None
    else:
        return headers_with_name[index]['headervalue']


def parse_email_header(email_headers, header_name):
    global NO_FETCH_EXTRACT
    header_value = get_header_value(email_headers, header_name=header_name)
    if header_value is None:
        email_address = email_domain = email_suffix = None
    else:
        email_address = parseaddr(header_value)[1]
        ext = NO_FETCH_EXTRACT(email_address)
        email_domain = ext.domain
        email_suffix = ext.suffix

    return {'name': header_name, 'address': email_address, 'domain': email_domain, 'suffix': email_suffix}


def extract_server_address(received_value):
    global RECEIVED_SERVER_REGEX, NO_FETCH_EXTRACT, IP_DOMAIN_TOKEN
    server_address_list = re.findall(RECEIVED_SERVER_REGEX, received_value)
    if len(server_address_list) == 0:
        server_domain = IP_DOMAIN_TOKEN
        server_suffix = None
    else:
        server_address = server_address_list[0]
        ext = NO_FETCH_EXTRACT(server_address)
        server_domain = ext.domain
        server_suffix = ext.suffix
    return server_domain, server_suffix


def extract_envelop_from_address(received_value):
    global ENVELOP_FROM_REGEX, NO_FETCH_EXTRACT
    from_envelop_address_list = re.findall(ENVELOP_FROM_REGEX, received_value)
    if len(from_envelop_address_list) == 0:
        from_envelop_address = from_envelop_domain = from_envelop_suffix = None
    else:
        from_envelop_address = from_envelop_address_list[0]
        from_envelop_address = parseaddr(from_envelop_address)[1]
        ext = NO_FETCH_EXTRACT(from_envelop_address)
        from_envelop_domain = ext.domain
        from_envelop_suffix = ext.suffix
    return from_envelop_address, from_envelop_domain, from_envelop_suffix


def parse_single_received_value(received_headers, index, name):
    if len(received_headers) >= index:
        received_value = received_headers[-index]['headervalue']
        server_domain, server_suffix = extract_server_address(received_value)
        from_envelop_address, from_envelop_domain, from_envelop_suffix = extract_envelop_from_address(received_value)
    else:
        server_domain = server_suffix = from_envelop_domain = from_envelop_address = from_envelop_suffix = None
    server_info = {'name': '{}-Server'.format(name), 'domain': server_domain, 'suffix': server_suffix}
    envelop_info = {'name': '{}-Envelope'.format(name), 'address': from_envelop_address, 'domain': from_envelop_domain,
                    'suffix': from_envelop_suffix}
    return server_info, envelop_info


def parse_received_headers(email_headers):
    received_headers = [header_dict for header_dict in email_headers if header_dict['headername'] == 'Received']
    n_received_headers = len(received_headers)
    first_server_info, first_envelop_info = parse_single_received_value(received_headers, 1, 'First-Received')
    second_server_info, second_envelop_info = parse_single_received_value(received_headers, 2, 'Second-Received')
    return n_received_headers, [first_server_info, first_envelop_info, second_server_info, second_envelop_info]


def get_rank_address(address):
    global DOMAIN_TO_RANK
    if address is None:
        return float('nan')
    if '@' in address:
        full_domain = address.split('@')[1]
    else:
        full_domain = address
    if full_domain in DOMAIN_TO_RANK:  # type: ignore
        return DOMAIN_TO_RANK[full_domain]  # type: ignore
    else:
        return -1


def compare_values(v1, v2):
    if v1 is None or v2 is None:
        return float('nan')
    return v1 == v2


def get_headers_features(email_headers):
    spf_result = get_header_value(email_headers, header_name='Received-SPF')
    if spf_result is not None:
        spf_result = spf_result.split()[0].lower()
    else:
        spf_result = VALIDATION_NA
    res_spf = {k: 0 for k in ['none', 'neutral', 'pass', 'fail', 'softfail', 'temperror', 'permerror', VALIDATION_NA]}
    if spf_result in res_spf:
        res_spf[spf_result] += 1
    else:
        res_spf[VALIDATION_OTHER] = 1
    res_spf['non-positive'] = spf_result in NON_POSITIVE_VALIDATION_VALUES
    authentication_results = get_header_value(email_headers, header_name='Authentication-Results')
    if authentication_results is not None:
        if 'dkim=' in authentication_results:
            dkim_result = authentication_results.split('dkim=')[-1].split()[0].lower().strip()
        else:
            dkim_result = VALIDATION_NA
    else:
        dkim_result = VALIDATION_NA
    res_dkim = {k: 0 for k in ['none', 'neutral', 'pass', 'fail', 'softfail', 'temperror', 'permerror', VALIDATION_NA]}
    if dkim_result in res_dkim:
        res_dkim[dkim_result] += 1
    else:
        res_dkim[VALIDATION_OTHER] = 1
    res_dkim['non-positive'] = spf_result in NON_POSITIVE_VALIDATION_VALUES
    list_unsubscribe = get_header_value(email_headers, header_name='List-Unsubscribe')
    unsubscribe_post = get_header_value(email_headers, header_name='List-Unsubscribe-Post-SPF')
    res_unsubscribe = 1 if list_unsubscribe is not None or unsubscribe_post is not None else 0
    from_dict = parse_email_header(email_headers, header_name='From')
    return_path_dict = parse_email_header(email_headers, header_name='Return-Path')
    reply_to_dict = parse_email_header(email_headers, header_name='Reply-To')
    n_received_headers, received_dicts_list = parse_received_headers(email_headers)
    all_addresses_dicts = received_dicts_list + [from_dict, return_path_dict, reply_to_dict]
    addresses_res = {}
    for a in all_addresses_dicts:
        addresses_res['{}::Exists'.format(a['name'])] = a['domain'] is not None
        if 'address' in a:
            addresses_res['{}::Rank'.format(a['name'])] = get_rank_address(a['address'])
        if 'Received' in a['name'] and 'Server' in a['name']:
            addresses_res['{}::IP_DOMAIN'.format(a['name'])] = a['domain'] == IP_DOMAIN_TOKEN
        addresses_res['{}::Suffix'.format(a['name'])] = a['suffix'] if a['suffix'] is not None else float('nan')

    for a1, a2 in combinations(all_addresses_dicts, 2):
        if 'address' in a1 and 'address' in a2:
            addresses_res['{}.Address=={}.Addres'.format(a1['name'], a2['name'])] = compare_values(a1['address'],
                                                                                                   a2['address'])
        addresses_res['{}.Domain=={}.Domain'.format(a1['name'], a2['name'])] = compare_values(a1['domain'],
                                                                                              a2['domain'])

    addresses_res['count_received'] = n_received_headers
    content_type_value = get_header_value(email_headers, header_name='Content-Type', index=0, ignore_case=True)
    if content_type_value is not None and ';' in content_type_value:
        content_type_value = content_type_value.split(';')[0]

    res = {}
    for k, v in res_spf.items():
        res['spf::{}'.format(k)] = v
    for k, v in res_dkim.items():
        res['dkim::{}'.format(k)] = v
    res['unsubscribe_headers'] = res_unsubscribe
    for k, v in addresses_res.items():
        res[k] = v  # type: ignore
    res['content-type'] = content_type_value
    return res


def get_attachments_features(email_attachments):
    res = {}
    res['number_of_attachments'] = len(email_attachments)
    all_attachments_names = []
    for a in email_attachments:
        attachment_name = a['name']
        if attachment_name is not None:
            all_attachments_names.append(attachment_name.lower())
    all_attachments_names_lengths = [len(name) for name in all_attachments_names]
    res['min_attachment_name_length'] = min(all_attachments_names_lengths) if len(
        all_attachments_names_lengths) > 0 else 0
    res['max_attachment_name_length'] = max(all_attachments_names_lengths) if len(
        all_attachments_names_lengths) > 0 else 0
    res['avg_attachment_name_length'] = float(sum(all_attachments_names_lengths)) / len(  # type: ignore
        all_attachments_names_lengths) if len(all_attachments_names_lengths) > 0 else 0  # type: ignore
    res['image_extension'] = 0
    res['raw_extensions'] = [name.split('.')[-1] for name in all_attachments_names]  # type: ignore
    for image_format in IMG_FORMATS:
        res['image_extension'] += sum([name.endswith(image_format) for name in all_attachments_names])
    res['txt_extension'] = sum([name.endswith('.txt') for name in all_attachments_names])
    res['exe_extension'] = sum([name.endswith('.exe') for name in all_attachments_names])
    res['archives_extension'] = sum(
        [name.endswith('.zip') or name.endswith('.rar') or name.endswith('.lzh') or name.endswith('.7z') for name in
         all_attachments_names])
    res['pdf_extension'] = sum([name.endswith('.pdf') for name in all_attachments_names])
    res['disk_img_extension'] = sum([name.endswith('.iso') or name.endswith('.img') for name in all_attachments_names])
    res['other_executables_extension'] = sum(
        [any(name.endswith(ext) for ext in ['.jar', '.bat', '.psc1', '.vb', '.vbs', '.msi', '.cmd', '.reg', '.wsf']) for
         name in all_attachments_names])

    res['office_extension'] = 0
    for offic_format in ['.doc', '.xls', '.ppt', 'xlsx', 'xlsm']:
        res['office_extension'] += sum([name.endswith(offic_format) for name in all_attachments_names])
    return res


def transform_text_to_ngrams_counter(email_body_word_tokenized, email_subject_word_tokenized):
    text_ngram = []
    for n in range(1, 4):
        text_ngram += list(ngrams(email_body_word_tokenized, n))
        text_ngram += list(ngrams(email_subject_word_tokenized, n))
    text_ngrams = Counter(text_ngram)
    return text_ngrams


def get_closing_fields_from_incident(row):
    if 'owner' in row:
        owner = row['owner']
        if owner not in ['admin', ''] and isinstance(owner, str):
            owner = hash_value(owner)
    else:
        owner = float('nan')
    if 'closingUserId' in row:
        closing_user = row['closingUserId']
        if closing_user not in ['admin', '', 'DBot'] and isinstance(closing_user, str):
            closing_user = hash_value(closing_user)
    else:
        closing_user = float('nan')
    if 'closeNotes' in row:
        close_notes = row['closeNotes']
        if isinstance(close_notes, str):
            close_notes = close_notes.strip().lower()
            close_notes_tokenized = word_tokenize(close_notes)
            close_notes_tokenized = [token if token in EMBEDDING_DICT_FASTTEXT else hash_value(token)  # type: ignore
                                     for token in close_notes_tokenized]  # type: ignore
            close_notes = ' '.join(close_notes_tokenized)

    else:
        close_notes = float('nan')
    return {'owner': owner, 'closing_user': closing_user, 'close_notes': close_notes}


def find_forwarded_features(email_subject, email_body):
    forwarded = response = False
    if re.search('- Forwarded message -', email_body, flags=re.IGNORECASE):
        forwarded = True
    forwarded_patterns = ['FW', 'Fwd']
    re_patterns = ['re']

    for pattern in forwarded_patterns:
        if re.search(r'(?<!\w)({})(?!\w)'.format(pattern), email_subject, flags=re.IGNORECASE):
            forwarded = True
            break
    for pattern in re_patterns:
        if re.search(r'(?<!\w)({})(?!\w)'.format(pattern), email_subject, flags=re.IGNORECASE):
            response = True
            break
    return {'forwarded': forwarded, 'response': response}


def clean_email_subject(email_subject):
    return re.sub(r"\[[^\]]*?\]", '', email_subject).strip()


def get_bert_features_for_text(text):
    global TOKENIZER, BERT_MODEL
    encoded_input = TOKENIZER(text, padding='max_length', max_length=512,  # type: ignore
                              return_tensors='np', truncation=True)
    ort_inputs = {
        'input_ids': encoded_input['input_ids'],
        "attention_mask": encoded_input['attention_mask'],
    }
    ort_outs = ORT_SESSION.run(None, ort_inputs)  # type: ignore
    first_hidden_state = ort_outs[0][0][0].tolist()
    return first_hidden_state


def get_bert_features_for_incident(email_subject, email_body):
    return {'first_vec_subject_body': get_bert_features_for_text(email_subject + '\n' + email_body)}


def extract_features_from_incident(row, label_fields):
    global EMAIL_BODY_FIELD, EMAIL_SUBJECT_FIELD, EMAIL_HTML_FIELD, EMAIL_ATTACHMENT_FIELD, EMAIL_HEADERS_FIELD
    email_body = row[EMAIL_BODY_FIELD] if EMAIL_BODY_FIELD in row else ''
    email_subject = row[EMAIL_SUBJECT_FIELD] if EMAIL_SUBJECT_FIELD in row else ''
    email_html = row[EMAIL_HTML_FIELD] if EMAIL_HTML_FIELD in row and isinstance(row[EMAIL_HTML_FIELD], str) else ''
    email_headers = row[EMAIL_HEADERS_FIELD] if \
        EMAIL_HEADERS_FIELD in row and isinstance(row[EMAIL_HEADERS_FIELD], list) else []
    email_attachments = row[EMAIL_ATTACHMENT_FIELD] if EMAIL_ATTACHMENT_FIELD in row else []
    email_attachments = email_attachments if email_attachments is not None else []
    if isinstance(email_html, float):
        email_html = ''
    if email_body is None or isinstance(email_body, float) or email_body.strip() == '':
        email_body = get_text_from_html(email_html)
    if isinstance(email_subject, float):
        email_subject = ''
    email_body, email_subject = email_body.strip().lower(), email_subject.strip().lower()
    email_subject = clean_email_subject(email_subject)
    text = email_subject + ' ' + email_body
    if len(text) < MIN_TEXT_LENGTH:
        raise ShortTextException('Text length is shorter than allowed minimum of: {}'.format(MIN_TEXT_LENGTH))
    email_body_word_tokenized = word_tokenize(email_body)
    email_subject_word_tokenized = word_tokenize(email_subject)
    text_ngrams = transform_text_to_ngrams_counter(email_body_word_tokenized, email_subject_word_tokenized)
    ngrams_features = get_ngrams_features(text, text_ngrams)
    soup = BeautifulSoup(email_html, "html.parser")

    lexical_features = get_lexical_features(email_subject, email_body, email_body_word_tokenized,
                                            email_subject_word_tokenized)
    characters_features = get_characters_features(text)
    html_feature = get_html_features(soup)
    ml_features = get_embedding_features(email_body_word_tokenized + email_subject_word_tokenized)
    ml_features_subject = get_embedding_features(email_subject_word_tokenized)
    ml_features_body = get_embedding_features(email_body_word_tokenized)
    bert_features = get_bert_features_for_incident(email_subject, email_body)
    headers_features = get_headers_features(email_headers)
    url_feautres = get_url_features(email_body=email_body, email_html=email_html, soup=soup)
    attachments_features = get_attachments_features(email_attachments=email_attachments)
    res = {
        'ngrams_features': ngrams_features,
        'lexical_features': lexical_features,
        'characters_features': characters_features,
        'html_feature': html_feature,
        'ml_features': ml_features,
        'ml_features_subject': ml_features_subject,
        'ml_features_body': ml_features_body,
        'headers_features': headers_features,
        'url_features': url_feautres,
        'attachments_features': attachments_features,
        'bert_features': bert_features,
        'created': str(row['created']) if 'created' in row else None,
        'id': str(row['id']) if 'id' in row else None,
        'type': str(row['type']) if 'type' in row else None,
    }
    for label in label_fields:
        if label in row:
            res[label] = row[label]
        else:
            res[label] = float('nan')
    try:
        res.update(get_closing_fields_from_incident(row))
    except Exception:
        pass
    try:
        res.update(find_forwarded_features(email_subject, email_body))
    except Exception:
        pass

    return res


def extract_features_from_all_incidents(incidents_df, label_fields):
    X = []
    exceptions_log = []
    exception_indices = set()
    timeout_indices = set()
    short_text_indices = set()
    durations = []
    for index, row in incidents_df.iterrows():
        signal.alarm(5)
        try:
            start = time.time()
            X_i = extract_features_from_incident(row, label_fields)
            end = time.time()
            X.append(X_i)
            durations.append(end - start)
        except TimeoutException:
            timeout_indices.add(index)
        except ShortTextException:
            short_text_indices.add(index)
        except Exception:
            exception_indices.add(index)
            exceptions_log.append(traceback.format_exc())
            if len(exception_indices) == MAX_ALLOWED_EXCEPTIONS:
                break
        finally:
            signal.alarm(0)
    return X, Counter(exceptions_log).most_common(), short_text_indices, exception_indices, timeout_indices, durations


def extract_data_from_incidents(incidents, input_label_field=None):
    incidents_df = pd.DataFrame(incidents)
    if 'created' in incidents_df:
        incidents_df['created'] = incidents_df['created'].apply(lambda x: dateutil.parser.parse(x))  # type: ignore
        incidents_df_for_finding_labels_fields_candidates = incidents_df.sort_values(by='created', ascending=False) \
            .head(500)
    else:
        incidents_df_for_finding_labels_fields_candidates = incidents_df
    if input_label_field is None:
        label_fields = find_label_fields_candidates(incidents_df_for_finding_labels_fields_candidates)
    else:
        input_label_field = input_label_field.strip()
        if input_label_field not in incidents_df:
            return_error('Could not find label field "{}" among the incidents'.format(input_label_field))
        label_fields = [input_label_field]
    for label in label_fields:
        incidents_df[label].replace('', float('nan'), regex=True, inplace=True)
    n_fetched_incidents = len(incidents_df)
    incidents_df = incidents_df.dropna(how='all', subset=label_fields).reset_index()
    n_incidents = len(incidents_df)
    y = []
    for i, label in enumerate(label_fields):
        y.append({'field_name': label,
                  'rank': '#{}'.format(i + 1)})
    custom_fields = [col for col in incidents_df.columns if col not in LABEL_FIELDS_BLACKLIST]
    custom_fields_dict = {}
    for col in custom_fields:
        try:
            custom_fields_dict[col] = float(incidents_df[col].nunique() / n_incidents)
        except Exception:
            pass
    subject_field_exists = EMAIL_SUBJECT_FIELD in incidents_df.columns
    body_field_exists = EMAIL_BODY_FIELD in incidents_df.columns
    html_field_exists = EMAIL_HTML_FIELD in incidents_df.columns
    headers_field_exists = EMAIL_HEADERS_FIELD in incidents_df.columns
    attachments_field_exists = EMAIL_ATTACHMENT_FIELD in incidents_df.columns
    n_missing_subject_field = sum(incidents_df[EMAIL_SUBJECT_FIELD].isnull()) if subject_field_exists else n_incidents
    n_missing_body_field = sum(incidents_df[EMAIL_BODY_FIELD].isnull()) if body_field_exists else n_incidents
    n_missing_html_field = sum(incidents_df[EMAIL_HTML_FIELD].isnull()) if html_field_exists else n_incidents
    n_missing_headers_field = sum(incidents_df[EMAIL_HEADERS_FIELD].isnull()) if headers_field_exists else n_incidents
    n_missing_attachments_field = sum(incidents_df[EMAIL_ATTACHMENT_FIELD].isnull()) \
        if attachments_field_exists else n_incidents
    if not (subject_field_exists or body_field_exists or html_field_exists):
        X = []
        exceptions_log = []
        short_text_indices = []
        exception_indices = []
        timeout_indices = []
        durations = []
    else:
        load_external_resources()
        X, exceptions_log, short_text_indices, exception_indices, timeout_indices, durations \
            = extract_features_from_all_incidents(incidents_df, label_fields)

    return {'X': X,
            'n_fetched_incidents': len(X),
            'y': y,
            'log':
                {'exceptions': exceptions_log,
                 'n_timout': len(timeout_indices),
                 'n_other_exceptions': len(exception_indices),
                 'durations': durations,
                 'n_fetched_incidents': n_fetched_incidents,
                 'n_labeled_incidents': n_incidents,
                 'subject_field_exists': subject_field_exists,
                 'body_field_exists': body_field_exists,
                 'html_field_exists': html_field_exists,
                 'headers_field_exists': headers_field_exists,
                 'attachments_field_exists': attachments_field_exists,
                 'n_missing_subject_field': n_missing_subject_field,
                 'n_missing_body_field': n_missing_body_field,
                 'n_missing_html_field': n_missing_html_field,
                 'n_missing_headers_field': n_missing_headers_field,
                 'n_missing_attachments_field': n_missing_attachments_field,
                 'n_short_text_fields': len(short_text_indices),
                 'custom_fields': custom_fields_dict
                 },
            }


def return_json_entry(obj):
    entry = {
        "Type": entryTypes["note"],
        "ContentsFormat": formats["json"],  # type: ignore
        "Contents": obj,
    }
    demisto.results(entry)


def get_args_based_on_last_execution():
    return {'limit': MAX_INCIDENTS_TO_FETCH_PERIODIC_EXECUTION,
            'fromDate': FROM_DATA_PERIODIC_EXECUTION}


def update_last_execution_time():
    execution_datetime_str = datetime.strftime(datetime.now(), DATETIME_FORMAT)
    list_content = json.dumps({EXECUTION_JSON_FIELD: execution_datetime_str, VERSION_JSON_FIELD: FETCH_DATA_VERSION})
    res = demisto.executeCommand("createList", {"listName": LAST_EXECUTION_LIST_NAME, "listData": list_content})
    if is_error(res):
        demisto.results(res)


def determine_incidents_args(input_args, default_args):
    get_incidents_by_query_args = {}
    if 'query' in input_args:
        get_incidents_by_query_args['query'] = '({}) and (status:Closed)'.format(input_args['query'])
    elif 'query' in default_args:
        get_incidents_by_query_args['query'] = '({}) and (status:Closed)'.format(default_args['query'])
    else:
        get_incidents_by_query_args['query'] = 'status:Closed'
    for arg in ['limit', 'fromDate', 'incidentTypes', 'toDate']:
        if arg in input_args:
            get_incidents_by_query_args[arg] = input_args[arg]
        elif arg in default_args:
            get_incidents_by_query_args[arg] = default_args[arg]
    return get_incidents_by_query_args


def set_incidents_fields_names(input_args):
    global EMAIL_BODY_FIELD, EMAIL_SUBJECT_FIELD, EMAIL_HTML_FIELD, EMAIL_HTML_FIELD, EMAIL_HEADERS_FIELD, \
        EMAIL_ATTACHMENT_FIELD
    EMAIL_BODY_FIELD = input_args.get('emailBody', EMAIL_BODY_FIELD)
    EMAIL_SUBJECT_FIELD = input_args.get('emailSubject', EMAIL_SUBJECT_FIELD)
    EMAIL_HTML_FIELD = input_args.get('emailBodyHTML', EMAIL_HTML_FIELD)
    EMAIL_HEADERS_FIELD = input_args.get('emailHeaders', EMAIL_HEADERS_FIELD)
    EMAIL_ATTACHMENT_FIELD = input_args.get('emailAttachments', EMAIL_ATTACHMENT_FIELD)


def return_file_entry(res, num_of_incidents):
    file_name = str(uuid.uuid4())
    entry = fileResult(file_name, json.dumps(res))
    entry['Contents'] = res
    entry['HumanReadable'] = 'Fetched features from {} incidents'.format(num_of_incidents)
    entry["ContentsFormat"]: formats["json"]  # type: ignore
    demisto.results(entry)


def main():
    input_args = demisto.args()
    set_incidents_fields_names(input_args)
    default_args = get_args_based_on_last_execution()
    get_incidents_by_query_args = determine_incidents_args(input_args, default_args)
    incidents_query_res = demisto.executeCommand('GetIncidentsByQuery', get_incidents_by_query_args)
    if is_error(incidents_query_res):
        return_error(get_error(incidents_query_res))
    incidents = json.loads(incidents_query_res[-1]['Contents'])
    if len(incidents) == 0:
        demisto.results('No results were found')
    else:
        tag_field = demisto.args().get('tagField', None)
        data = extract_data_from_incidents(incidents, tag_field)
        data_str = json.dumps(data)
        compress = demisto.args().get('compress', 'True') == 'True'
        if compress:
            encoded_data = data_str.encode('utf-8', errors='ignore')
            compressed_data = zlib.compress(encoded_data, 4)
            compressed_hr_data = b64encode(compressed_data).decode('utf-8')
        else:
            compressed_hr_data = data_str
        res = {'PayloadVersion': FETCH_DATA_VERSION, 'PayloadData': compressed_hr_data,
               'ExecutionTime': datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), 'IsCompressed': compress}
        return_file = input_args.get('toFile', 'False').strip() == 'True'
        if return_file:
            return_file_entry(res, len(data['X']))
        else:
            return_json_entry(res)


if __name__ in ['__main__', '__builtin__', 'builtins']:
    main()