Merge remote-tracking branch 'origin/dev' into dev

.gitignore

@@ -109,3 +109,5 @@ venv
 .vscode/*
 
 *.db
+
+start.py

.travis.yml

@@ -5,6 +5,7 @@ python:
   - "3.5"
   - "3.6"
   - "3.7"
+
 # command to install dependencies
 install:
   - pip install -r requirements.txt
@@ -15,8 +16,9 @@ script:
   - python -m compileall ./
   - coverage run --omit='*/virtualenv/*,*/site-packages/*' -m unittest discover -s . -v -p "*_test.py"
 
+# create coverage for one python version
 after_success:
-  - coveralls
+  - test $TRAVIS_PYTHON_VERSION == "3.7" && coveralls
 
 notifications:
   email: false

docker-compose.yml

@@ -0,0 +1,25 @@
+version: '3'
+services:
+  pastepwn:
+    build:
+      context: . 
+    env_file:
+     - .env
+    environment:
+      - HTTP_PROXY=${HTTP_PROXY}
+      - HTTPS_PROXY=${HTTPS_PROXY}
+      - NO_PROXY=${NO_PROXY}
+      - PATH_TO_START_PY=${PATH_TO_START_PY}
+    volumes:
+      - "${PATH_TO_START_PY}:/pastepwn/start.py:ro"
+    depends_on:
+      - db
+    links:
+      - db
+  db:
+    image: mysql
+    env_file:
+      - .env
+    ports:
+      - "3306:3306"
+

pastepwn/actions/__init__.py

@@ -3,8 +3,10 @@
 from .basicaction import BasicAction
 from .databaseaction import DatabaseAction
 from .discordaction import DiscordAction
+from .emailaction import EmailAction
 from .genericaction import GenericAction
 from .logaction import LogAction
+from .mispaction import MISPAction
 from .savefileaction import SaveFileAction
 from .savejsonaction import SaveJSONAction
 from .telegramaction import TelegramAction
@@ -20,4 +22,6 @@
     "SaveJSONAction",
     "TwitterAction",
     "DiscordAction",
+    "MISPAction",
+    "EmailAction"
 )

pastepwn/actions/emailaction.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+import re
+import smtplib
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+
+from .basicaction import BasicAction
+from pastepwn.util import TemplatingEngine
+
+
+class EmailAction(BasicAction):
+    """This action takes a username, password, receiver mail address,
+    hostname, port and when executed sends out an e-mail to the receiver
+    containing the paste"""
+    name = "EmailAction"
+
+    def __init__(self, username, password, receiver, hostname, port=465, template=None):
+        super().__init__()
+        mail_regex = r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)"
+        if username is None or not re.match(mail_regex, username):
+            raise ValueError('Invalid username !')
+        else:
+            self.username = username
+        if receiver is None or not re.match(mail_regex, receiver):
+            raise ValueError('Invalid reciever address !')
+        else:
+            self.receiver = receiver
+        self.password = password
+        self.hostname = hostname
+        self.port = port
+        self.template = template
+
+    def perform(self, paste, analyzer_name=None):
+        """
+        Sends an email to the specified receiver with the paste's content
+        :param paste: The paste passed by the ActionHandler
+        :param analyzer_name: The name of the analyzer which matched the paste
+        :return: None
+        """
+        text = TemplatingEngine.fill_template(paste, analyzer_name, template_string=self.template)
+
+        email = MIMEMultipart()
+        email['From'] = self.username
+        email['To'] = self.receiver
+        email['Subject'] = 'Paste matched by pastepwn via analyzer "{}"'.format(analyzer_name)
+        email.attach(MIMEText(text, 'plain'))
+
+        # TODO there should be a way to use starttls - check https://realpython.com/python-send-email/
+        with smtplib.SMTP_SSL(self.hostname, self.port) as smtp:
+            smtp.login(self.username, self.password)
+            text = email.as_string()
+            smtp.sendmail(self.username, self.receiver, text)

pastepwn/actions/mispaction.py

@@ -0,0 +1,120 @@
+# -*- coding: utf-8 -*-
+import json
+import logging
+import time
+from pastepwn.util import Request
+from .basicaction import BasicAction
+
+
+class MISPAction(BasicAction):
+    """
+    Action to add an event to a MISP instance on a matched paste
+    
+    Documentation for adding events:
+    https://www.circl.lu/doc/misp/automation/#post-events
+
+    The MISPAction objects can take a `transformer` function as a constructor parameter.
+    This function (by default MISPAction.default_transformer) should take a Paste and an
+    optional analyzer name as parameters (just like BasicAction.perform), and return a
+    dictionary representing a MISP event, which will then be sent to the API.
+
+    Additional attributes can be sent with each event, specified by the `attributes`
+    parameter. Here is the documentation regarding types and categories:
+    https://www.circl.lu/doc/misp/categories-and-types/
+    """
+    name = "MISPAction"
+
+    def __init__(self, url, access_key, transformer=None, attributes=None):
+        """
+        Init method for the MISPAction
+        :param url:         string      URL of the MISP instance (complete with protocol and port)
+        :param access_key:  string      MISP access key for authorization
+        :param transformer: Callable    Takes a Paste (and optional analyzer name) as parameter 
+                                        and returns a MISP-formatted event as a dictionary
+        :param attributes:  Iterable    List of fully defined attributes to add to events
+        """
+        super().__init__()
+        self.logger = logging.getLogger(__name__)
+        self.url = url
+        self.access_key = access_key
+        if transformer is None:
+            self.transformer = MISPAction.default_transformer
+        else:
+            self.transformer = transformer
+        self.attributes = attributes
+
+    @staticmethod
+    def default_transformer(paste, analyzer_name=None):
+        timestamp = time.gmtime(int(paste.date))
+        attrs = []
+        # Build event
+        event = {
+            "date": time.strftime('%Y-%m-%d' , timestamp),
+            "info":"Sensitive information found on pastebin (type: %s)" % analyzer_name,
+            "threat_level_id": 4,   # Undefined
+            "published": False,     # Unpublished
+            "analysis": 0,          # Not yet analyzed
+            "distribution": 0,      # Shared with organization only
+            "Attribute": []
+        }
+        # Add link to the paste
+        attrs.append({
+            "type": "url",
+            "category": "Network activity",
+            "comment": "Link to pastebin paste containing information",
+            "value": paste.full_url
+        })
+        # Add username of the author
+        attrs.append({
+            "type": "text",
+            "category": "Attribution",
+            "comment": "Username of paste author",
+            "value": paste.user
+        })
+        # Add size of the paste
+        attrs.append({
+            "type": "size-in-bytes",
+            "category": "Other",
+            "comment": "Size of the paste",
+            "value": paste.size
+        })
+        # Attach full paste if it's small
+        if int(paste.size) <= 1024 and paste.body is not None:
+            attrs.append({
+                "type": "attachment",
+                "category": "Artifacts dropped",
+                "comment": "Raw body of the paste",
+                "value": paste.body
+            })
+        # Add attributes to the event
+        event['Attribute'] = attrs
+        return event
+
+    def perform(self, paste, analyzer_name=None):
+        """
+        Sends the event to the MISP instance.
+        :param paste: The paste passed by the ActionHandler
+        :param analyzer_name: The name of the analyzer which matched the paste
+        """
+        # Call transformer to construct payload
+        event = self.transformer(paste, analyzer_name)
+        if self.attributes:
+            # Add extra attributes
+            event['Attributes'].extend(self.attributes)
+        data = json.dumps({"Event": event})
+        # Send event to MISP instance
+        r = Request()
+        r.headers = {'Authorization': self.access_key, 'Accept': 'application/json', 'Content-Type': 'application/json'}
+        res = r.post(self.url + "/events", data=data)
+        # Error handling
+        if not res:
+            self.logger.warning("Empty response when adding event")
+        else:
+            res = json.loads(res)
+            if 'Event' in res:
+                self.logger.info('Event #%s successfully added to MISP', res['Event']['id'])
+            else:
+                # An error has happened, but the 'errors' field is not always present
+                if 'errors' in res:
+                    self.logger.error('Error when adding event: %s', res['errors'])
+                self.logger.warning('Failed to add event: %s', res.get('message'))

pastepwn/analyzers/__init__.py

@@ -4,6 +4,7 @@
 from .awsaccesskeyanalyzer import AWSAccessKeyAnalyzer
 from .awssecretkeyanalyzer import AWSSecretKeyAnalyzer
 from .awssessiontokenanalyzer import AWSSessionTokenAnalyzer
+from .azuresubscriptionkeyanalyzer import AzureSubscriptionKeyAnalyzer
 from .base64analyzer import Base64Analyzer
 from .basicanalyzer import BasicAnalyzer
 from .battlenetkeyanalyzer import BattleNetKeyAnalyzer
@@ -14,7 +15,9 @@
 from .emailpasswordpairanalyzer import EmailPasswordPairAnalyzer
 from .facebookaccesstokenanalyzer import FacebookAccessTokenAnalyzer
 from .genericanalyzer import GenericAnalyzer
+from .googleapikeyanalyzer import GoogleApiKeyAnalyzer
 from .googleoauthkeyanalyzer import GoogleOAuthKeyAnalyzer
+from .hashanalyzer import HashAnalyzer
 from .ibananalyzer import IBANAnalyzer
 from .logicalanalyzers import AndAnalyzer
 from .logicalanalyzers import LogicalBaseAnalyzer
@@ -45,6 +48,7 @@
     'AWSSecretKeyAnalyzer',
     'AWSAccessKeyAnalyzer',
     'AWSSessionTokenAnalyzer',
+    'AzureSubscriptionKeyAnalyzer',
     'Base64Analyzer',
     'BasicAnalyzer',
     'BattleNetKeyAnalyzer',
@@ -55,7 +59,9 @@
     'EmailPasswordPairAnalyzer',
     'FacebookAccessTokenAnalyzer',
     'GenericAnalyzer',
+    'GoogleApiKeyAnalyzer',
     'GoogleOAuthKeyAnalyzer',
+    'HashAnalyzer',  
     'IBANAnalyzer',
     'LogicalBaseAnalyzer',
     'MailAnalyzer',

pastepwn/analyzers/azuresubscriptionkeyanalyzer.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+from .regexanalyzer import RegexAnalyzer
+
+
+class AzureSubscriptionKeyAnalyzer(RegexAnalyzer):
+    """
+    Analyzer to match Azure subscription key via regex
+
+    Keys are 32 character alphanumeric (lower case)
+    """
+    name = "AzureSubscriptionKeyAnalyzer"
+
+    def __init__(self, actions):
+        # https://docs.microsoft.com/en-us/azure/api-management/api-management-subscriptions
+        regex = r"\b[a-f0-9]{32}\b"
+        super().__init__(actions, regex)

pastepwn/analyzers/googleapikeyanalyzer.py

@@ -0,0 +1,10 @@
+# -*- coding: utf-8 -*-
+from .regexanalyzer import RegexAnalyzer
+
+
+class GoogleApiKeyAnalyzer(RegexAnalyzer):
+
+    def __init__(self, actions):
+        # https://cloud.google.com/docs/authentication/api-keys
+        regex = r"\bAIza[0-9A-Za-z_-]{35}\b"
+        super().__init__(actions, regex)

pastepwn/analyzers/hashanalyzer.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+import hashlib
+import re
+from .regexanalyzer import RegexAnalyzer
+
+
+class HashAnalyzer(RegexAnalyzer):
+    """Analyzer to match multiple hashes of user-given passwords."""
+    name = "HashAnalyzer"
+
+    def __init__(self, actions, passwords, algorithms=None):
+        """"Hashes given passwords with multiple algorithms and matches the output.
+
+        :param actions: A single action or a list of actions to be executed on every paste
+        :param passwords: A single password or a list of passwords to hash, as bytes
+        :param algorithms: A list of algorithm names to use for hashing. This should be a subset
+                           of hashlib.algorithms_available, and defaults to it.
+        """
+        # Make sure passwords is a list
+        if isinstance(passwords, bytes):
+            passwords = [passwords]
+
+        # Build algorithm list
+        if algorithms is None:
+            algorithms = hashlib.algorithms_available
+        else:
+            algorithms = set(algorithms).intersection(hashlib.algorithms_available)
+
+        if not algorithms:
+            raise ValueError('No valid algorithm names specified')
+
+        # Compute hashes with all algorithms
+        hashes = []
+        for hash_name in algorithms:
+            hash_function = hashlib.new(hash_name)
+            for password in passwords:
+                hash = hash_function.copy()
+                hash.update(password)
+                if hash_name == 'shake_128':
+                    digest = hash.hexdigest(128)
+                elif hash_name == 'shake_256':
+                    digest = hash.hexdigest(256)
+                else:
+                    digest = hash.hexdigest()
+                hashes.append(digest)
+
+        # Build regex
+        regex = r"\b(%s)\b" % '|'.join(hashes)
+        super().__init__(actions, regex, re.IGNORECASE)

pastepwn/analyzers/regexanalyzer.py

@@ -8,9 +8,9 @@ class RegexAnalyzer(BasicAnalyzer):
     """Analyzer to match the content of a paste via regular expressions"""
     name = "RegexAnalyzer"
 
-    def __init__(self, actions, regex):
+    def __init__(self, actions, regex, flags=0):
         super().__init__(actions)
-        self.regex = re.compile(regex)
+        self.regex = re.compile(regex, flags)
 
     def match(self, paste):
         """Match the content of a paste via regex. Return true if regex matches"""