Merge branch 'master' of github.com:MISP/PyMISP

.travis.yml

@@ -9,27 +9,13 @@ addons:
       - libstdc++6
       - libfuzzy-dev
 
-
-matrix:
-    include:
-        - name: "Python 2.7 - legacy"
-          python: 2.7
-          env: LEGACY=true
-        - name: "Python 3.5"
-          python: 3.5
-          dist: xenial
-        - name: "Python 3.6"
-          python: 3.6
-          dist: xenial
-        - name: "Python 3.6 - Dev"
-          python: 3.6-dev
-          dist: xenial
-        - name: "Python 3.7"
-          python: 3.7
-          dist: xenial
-        - name: "Python 3.7 - Dev"
-          python: 3.7-dev
-          dist: xenial
+python:
+  - "3.6"
+  - "3.6-dev"
+  - "3.7"
+  - "3.7-dev"
+  - "3.8"
+  - "3.8-dev"
 
 install:
     - bash travis/install_travis.sh

CHANGELOG.txt

@@ -2,6 +2,71 @@ Changelog
 =========
 
 
+v2.4.120 (2020-01-17)
+---------------------
+
+New
+~~~
+- [attribute type] kusto-query attribute type. [Alexandre Dulaunoy]
+
+  Kusto query is the query language for the Kusto services in Azure used
+  to search large dataset. It's used in Windows Defender ATP Hunting-Queries
+  and also Azure Sentinel (Cloud-native SIEM).
+- Remove python < 3.6 support. [Raphaël Vinot]
+
+Changes
+~~~~~~~
+- Bump version. [Raphaël Vinot]
+- Bump Changelog. [Raphaël Vinot]
+- Bump misp-objects. [Raphaël Vinot]
+- Bump dependencies, add debug. [Raphaël Vinot]
+- Upate dummy events creator. [Raphaël Vinot]
+- Add tests on more version of Python. [Raphaël Vinot]
+- Search with the STIX output returns a json STIX. [Raphaël Vinot]
+
+  Was XML before.
+- Bump dependencies. [Raphaël Vinot]
+- Add more typing information. [Raphaël Vinot]
+- Add typing markup. [Raphaël Vinot]
+- Bump misp-objects. [Raphaël Vinot]
+- Bump Dependencies. [Raphaël Vinot]
+- Bump misp-objects. [Raphaël Vinot]
+
+Fix
+~~~
+- Add missing variable in dummy creator. [Raphaël Vinot]
+- Et2misp was python2 only. [Raphaël Vinot]
+- Feed generator was broken. [Raphaël Vinot]
+
+  Fix #506
+- Event without hashable attribute. [Raphaël Vinot]
+
+  Related #506
+
+Other
+~~~~~
+- Update api.py. [AaronK]
+
+  minor typo, can;t help it noticing those. sorry,
+- Fixed TODO, added quarantineFolder/quarantineRule from
+  messagesBlocked, added some error handling to prevent empty attributes
+  from trying to be added. [th3jiv3r]
+- Scrape proofpoint tap api for messages blocked/delivered & clicks
+  blocked/permitted  and create misp events. [th3jiv3r]
+- Add variable for proofpoint tap api auth. [th3jiv3r]
+- Update README.md. [AaronK]
+
+  minor typo
+- Define the number of entries to output. [AndreC10002]
+
+  Allow for defining in the settings.py file the number of entries to output
+- Update generate.py. [AndreC10002]
+- Cleanup of code and 'quick-n-dirty' sanitizing of tags. [Koen Van
+  Impe]
+- Sync. [Koen Van Impe]
+- Update README.md. [Raphaël Vinot]
+
+
 v2.4.119.1 (2019-12-17)
 -----------------------
 
@@ -11,6 +76,7 @@ New
 
 Changes
 ~~~~~~~
+- Bump changelog. [Raphaël Vinot]
 - Version bump. [Raphaël Vinot]
 - Bump test files. [Raphaël Vinot]
 - Bump misp-objects. [Raphaël Vinot]

Pipfile

@@ -11,6 +11,7 @@ requests-mock = "*"
 pymisp = {editable = true,extras = ["fileobjects", "neo", "openioc", "virustotal", "pdfexport", "docs"],path = "."}
 docutils = "==0.15"
 memory-profiler = "*"
+mypy = "*"
 
 [packages]
 pymisp = {editable = true,extras = ["fileobjects", "openioc", "virustotal", "pdfexport"],path = "."}

docs/tutorial/FullOverview.ipynb

@@ -192,7 +192,7 @@
    "source": [
     "## Set parameters (inline)\n",
     "\n",
-    "This is the was to pass other parameters"
+    "This is the way to pass other parameters"
    ]
   },
   {
@@ -603,7 +603,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Use locally defined objet templates\n",
+    "## Use locally defined object templates\n",
     "\n",
     "**Important**: The path you pass as parameter for `misp_objects_path_custom` needs to contain a directory equals to the value of the parameter `name` (same structure as the content of the `misp-object` repository)\n"
    ]
@@ -654,7 +654,7 @@
    "source": [
     "## Use lief to extract indicators out of binaries\n",
     "\n",
-    "An other cool helper: one liner to whom you can pass the path to a binary, if it is supported by `lief` (PE/ELF/Mach-o), you get the the file object, a PE, ELF, or Mach-o object, and the relevant sections.\n",
+    "An other cool helper: one liner to whom you can pass the path to a binary, if it is supported by `lief` (PE/ELF/Mach-o), you get the file object, a PE, ELF, or Mach-o object, and the relevant sections.\n",
     "\n",
     "If it is anything else, it will just generate the the file object.\n"
    ]

examples/et2misp.py

@@ -71,7 +71,7 @@ def update_et_event(name):
         for k,v in et_attr.items():
             r = mymisp.delete_attribute(v)
             if r.get('errors'):
-                print "Error deleting attribute {} ({}): {}\n".format(v,k,r['errors'])
+                print("Error deleting attribute {} ({}): {}\n".format(v,k,r['errors']))
 
         # Weed out ips already in the MISP event
         for k,v in et_ips.items():
@@ -102,9 +102,9 @@ def update_et_event(name):
 def echeck(r, eid=None):
     if r.get('errors'):
         if eid:
-            print "Processing event {} failed: {}".format(eid, r['errors'])
+            print("Processing event {} failed: {}".format(eid, r['errors']))
         else:
-            print r['errors']
+            print(r['errors'])
         sys.exit(1)
 
 if __name__ == '__main__':

examples/events/create_massive_dummy_events.py

@@ -4,9 +4,11 @@
 from pymisp import ExpandedPyMISP
 try:
     from keys import url, key
+    verifycert = False
 except ImportError:
-    url = 'http://localhost:8080'
-    key = '8h0gHbhS0fv6JUOlTED0AznLXFbf83TYtQrCycqb'
+    url = 'https://localhost:8443'
+    key = 'd6OmdDFvU3Seau3UjwvHS1y3tFQbaRNhJhDX0tjh'
+    verifycert = False
 import argparse
 import tools
 
@@ -17,7 +19,7 @@
     parser.add_argument("-a", "--attribute", type=int, help="Number of attributes per event (default 3000)")
     args = parser.parse_args()
 
-    misp = ExpandedPyMISP(url, key, True)
+    misp = ExpandedPyMISP(url, key, verifycert)
     misp.toggle_global_pythonify()
 
     if args.limit is None:

examples/events/tools.py

@@ -4,7 +4,7 @@
 import random
 from random import randint
 import string
-from pymisp import MISPEvent
+from pymisp import MISPEvent, MISPAttribute
 
 
 def randomStringGenerator(size, chars=string.ascii_lowercase + string.digits):
@@ -15,65 +15,54 @@ def randomIpGenerator():
     return str(randint(0, 255)) + '.' + str(randint(0, 255)) + '.' + str(randint(0, 255)) + '.' + str(randint(0, 255))
 
 
+def _attribute(category, type, value):
+    attribute = MISPAttribute()
+    attribute.category = category
+    attribute.type = type
+    attribute.value = value
+    return attribute
+
+
 def floodtxt(misp, event, maxlength=255):
     text = randomStringGenerator(randint(1, maxlength))
-    textfunctions = [misp.add_internal_comment, misp.add_internal_text, misp.add_internal_other, misp.add_email_subject, misp.add_mutex, misp.add_filename]
-    textfunctions[randint(0, 5)](event, text)
+    choose_from = [('Internal reference', 'comment', text), ('Internal reference', 'text', text),
+                   ('Internal reference', 'other', text), ('Network activity', 'email-subject', text),
+                   ('Artifacts dropped', 'mutex', text), ('Artifacts dropped', 'filename', text)]
+    misp.add_attribute(event, _attribute(*random.choice(choose_from)))
 
 
 def floodip(misp, event):
     ip = randomIpGenerator()
-    ipfunctions = [misp.add_ipsrc, misp.add_ipdst]
-    ipfunctions[randint(0, 1)](event, ip)
+    choose_from = [('Network activity', 'ip-src', ip), ('Network activity', 'ip-dst', ip)]
+    misp.add_attribute(event, _attribute(*random.choice(choose_from)))
 
 
 def flooddomain(misp, event, maxlength=25):
     a = randomStringGenerator(randint(1, maxlength))
     b = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase)
     domain = a + '.' + b
-    domainfunctions = [misp.add_hostname, misp.add_domain]
-    domainfunctions[randint(0, 1)](event, domain)
-
-
-def flooddomainip(misp, event, maxlength=25):
-    a = randomStringGenerator(randint(1, maxlength))
-    b = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase)
-    domain = a + '.' + b
-    ip = randomIpGenerator()
-    misp.add_domain_ip(event, domain, ip)
+    choose_from = [('Network activity', 'domain', domain), ('Network activity', 'hostname', domain)]
+    misp.add_attribute(event, _attribute(*random.choice(choose_from)))
 
 
 def floodemail(misp, event, maxlength=25):
     a = randomStringGenerator(randint(1, maxlength))
     b = randomStringGenerator(randint(1, maxlength))
     c = randomStringGenerator(randint(2, 3), chars=string.ascii_lowercase)
     email = a + '@' + b + '.' + c
-    emailfunctions = [misp.add_email_src, misp.add_email_dst]
-    emailfunctions[randint(0, 1)](event, email)
-
-
-def floodattachment(misp, eventid, distribution, to_ids, category, comment, info, analysis, threat_level_id):
-    filename = randomStringGenerator(randint(1, 128))
-    misp.upload_sample(filename, 'dummy', eventid, distribution, to_ids, category, comment, info, analysis, threat_level_id)
+    choose_from = [('Network activity', 'email-dst', email), ('Network activity', 'email-src', email)]
+    misp.add_attribute(event, _attribute(*random.choice(choose_from)))
 
 
 def create_dummy_event(misp):
-    event = misp.new_event(0, 4, 0, 'dummy event')
-    flooddomainip(misp, event)
-    floodattachment(misp, event['Event']['id'], event['Event']['distribution'], False, 'Payload delivery', '', event['Event']['info'], event['Event']['analysis'], event['Event']['threat_level_id'])
+    return misp.new_event(0, 4, 0, 'dummy event')
 
 
 def create_massive_dummy_events(misp, nbattribute):
     event = MISPEvent()
     event.info = 'massive dummy event'
     event = misp.add_event(event)
     print(event)
-    eventid = event.id
-    distribution = '0'
-    functions = [floodtxt, floodip, flooddomain, flooddomainip, floodemail, floodattachment]
+    functions = [floodtxt, floodip, flooddomain, floodemail]
     for i in range(nbattribute):
-        choice = randint(0, 5)
-        if choice == 5:
-            floodattachment(misp, eventid, distribution, False, 'Payload delivery', '', event.info, event.analysis, event.threat_level_id)
-        else:
-            functions[choice](misp, event)
+        functions[random.randint(0, len(functions) - 1)](misp, event)

examples/feed-generator/README.md

@@ -7,7 +7,7 @@ This python script can be used to generate a MISP feed based on an existing MISP
 ````
 git clone https://github.com/MISP/PyMISP.git
 cd examples/feed-generator
-cp settings-default.py settings.py
+cp settings.default.py settings.py
 vi settings.py #adjust your settings
 python3 generate.py
 ````

examples/feed-generator/generate.py

@@ -5,7 +5,7 @@
 import json
 import os
 from pymisp import ExpandedPyMISP
-from settings import url, key, ssl, outputdir, filters, valid_attribute_distribution_levels
+from settings import entries, url, key, ssl, outputdir, filters, valid_attribute_distribution_levels
 
 valid_attribute_distributions = []
 
@@ -52,7 +52,7 @@ def saveManifest(manifest):
 if __name__ == '__main__':
     misp = init()
     try:
-        events = misp.search(metadata=True, limit=200, **filters, pythonify=True)
+        events = misp.search(metadata=True, limit=entries, **filters, pythonify=True)
     except Exception as e:
         print(e)
         sys.exit("Invalid response received from MISP.")

examples/feed-generator/settings.default.py

@@ -12,6 +12,9 @@
 # sure that you use a directory dedicated to the feed
 outputdir = 'output'
 
+# Determine the number of entries to output
+entries = 200
+
 # The filters to be used for by the feed. You can use any filter that
 # you can use on the event index, such as organisation, tags, etc.
 # It uses the same joining and condition rules as the API parameters

examples/keys.py.sample

@@ -5,3 +5,4 @@ misp_url = 'https://<your MISP URL>/'
 misp_key = 'Your MISP auth key' # The MISP auth key can be found on the MISP web interface under the automation section
 misp_verifycert = True
 misp_client_cert = ''
+proofpoint_key = 'Your Proofpoint TAP auth key'