Implement AWS Session Token Analyzer #122

examples/example.py

@@ -7,7 +7,6 @@
 from pastepwn.actions import TelegramAction
 from pastepwn.analyzers import MailAnalyzer, WordAnalyzer
 from pastepwn.database import MongoDB
-from pastepwn.scraping.pastebin import PastebinScraper
 
 # Setting up the logging
 logdir_path = os.path.dirname(os.path.abspath(__file__))
@@ -25,7 +24,6 @@
 database = MongoDB(ip="192.168.240.128")
 
 pastepwn = PastePwn(database)
-pastepwn.add_scraper(PastebinScraper())
 
 # Generic action to send Telegram messages on new matched pastes
 telegram_action = TelegramAction(token="token", receiver="-1001348376474")

pastepwn/actions/__init__.py

@@ -8,6 +8,7 @@
 from .databaseaction import DatabaseAction
 from .savejsonaction import SaveJSONAction
 from .twitteraction import TwitterAction
+from .discordaction import DiscordAction
 
 __all__ = (
     "BasicAction",
@@ -18,4 +19,5 @@
     "DatabaseAction",
     "SaveJSONAction",
     "TwitterAction",
+    "DiscordAction",
 )

pastepwn/actions/discordaction.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+import asyncio
+import json
+import logging
+import sys
+from string import Template
+
+from pastepwn.util import Request, DictWrapper
+from .basicaction import BasicAction
+
+
+class DiscordAction(BasicAction):
+    """Action to send a Discord message to a certain webhook or channel."""
+    name = "DiscordAction"
+
+    def __init__(self, webhook=None, token=None, channel_id=None, template=None):
+        super().__init__()
+        self.logger = logging.getLogger(__name__)
+        self.bot_available = True
+
+        try:
+            import websockets
+        except ImportError:
+            self.logger.warning("Could not import 'websockets' module. So you can only use webhooks for discord.")
+            self.bot_available = False
+
+        self.webhook = webhook
+        if webhook is None:
+            if token is None or channel_id is None:
+                raise ValueError('Invalid arguments: requires either webhook or token+channel_id arguments')
+
+            if not self.bot_available:
+                raise NotImplementedError("You can't use bot functionality without the 'websockets' module. Please import it or use webhooks!")
+
+            self.token = token
+            self.channel_id = channel_id
+            self.identified = False
+
+        if template is not None:
+            self.template = Template(template)
+        else:
+            self.template = None
+
+    @asyncio.coroutine
+    def _identify(self, ws_url):
+        """Connect to the Discord Gateway to identify the bot."""
+        # Docs: https://discordapp.com/developers/docs/topics/gateway#connecting-to-the-gateway
+        # Open connection to the Discord Gateway
+        socket = yield from websockets.connect(ws_url + '/?v=6&encoding=json')
+        try:
+            # Receive Hello
+            hello_str = yield from socket.recv()
+            hello = json.loads(hello_str)
+            if hello.get('op') != 10:
+                self.logger.warning('[ws] Expected Hello payload but received %s', hello_str)
+
+            # Send heartbeat and receive ACK
+            yield from socket.send(json.dumps({"op": 1, "d": {}}))
+            ack_str = yield from socket.recv()
+            ack = json.loads(ack_str)
+            if ack.get('op') != 11:
+                self.logger.warning('[ws] Expected Heartbeat ACK payload but received %s', ack_str)
+
+            # Identify
+            payload = {
+                "token": self.token,
+                "properties": {
+                    "$os": sys.platform,
+                    "$browser": "pastepwn",
+                    "$device": "pastepwn"
+                }
+            }
+            yield from socket.send(json.dumps({"op": 2, "d": payload}))
+
+            # Receive READY event
+            ready_str = yield from socket.recv()
+            ready = json.loads(ready_str)
+            if ready.get('t') != 'READY':
+                self.logger.warning('[ws] Expected READY event but received %s', ready_str)
+        finally:
+            # Close websocket connection
+            yield from socket.close()
+
+    def initialize_gateway(self):
+        """Initialize the bot token so Discord identifies it properly."""
+        if self.webhook is not None:
+            raise NotImplementedError('Gateway initialization is only necessary for bot accounts.')
+
+        # Call Get Gateway Bot to get the websocket URL
+        r = Request()
+        r.headers = {'Authorization': 'Bot {}'.format(self.token)}
+        res = json.loads(r.get('https://discordapp.com/api/gateway/bot'))
+        ws_url = res.get('url')
+
+        # Start websocket client
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        loop.run_until_complete(self._identify(ws_url))
+        self.identified = True
+
+    def perform(self, paste, analyzer_name=None):
+        """Send a message via Discord to a specified channel, without checking for errors"""
+        r = Request()
+        if self.template is None:
+            text = "New paste matched by analyzer '{0}' - Link: {1}".format(analyzer_name, paste.full_url)
+        else:
+            paste_dict = paste.to_dict()
+            paste_dict["analyzer_name"] = analyzer_name
+            text = self.template.safe_substitute(DictWrapper(paste_dict))
+
+        if self.webhook is not None:
+            # Send to a webhook (no authentication)
+            url = self.webhook
+        else:
+            # Send through Discord bot API (header-based authentication)
+            url = 'https://discordapp.com/api/channels/{0}/messages'.format(self.channel_id)
+            r.headers = {'Authorization': 'Bot {}'.format(self.token)}
+
+        res = r.post(url, {"content": text})
+        if res == "":
+            # If the response is empty, skip further execution
+            return
+
+        res = json.loads(res)
+
+        if res.get('code') == 40001 and self.bot_available and self.webhook is None and not self.identified:
+            # Unauthorized access, bot token hasn't been identified to Discord Gateway
+            self.logger.info('Accessing Discord Gateway to initialize token')
+            self.initialize_gateway()
+            # Retry action
+            self.perform(paste, analyzer_name=analyzer_name)

pastepwn/analyzers/__init__.py

@@ -1,17 +1,34 @@
 # -*- coding: utf-8 -*-
 from .alwaystrueanalyzer import AlwaysTrueAnalyzer
 from .basicanalyzer import BasicAnalyzer
+from .battlenetkeyanalyzer import BattleNetKeyAnalyzer
 from .bcrypthashanalyzer import BcryptHashAnalyzer
-from .md5hashanalyzer import MD5HashAnalyzer
-from .shahashanalyzer import SHAHashAnalyzer
 from .creditcardanalyzer import CreditCardAnalyzer
+from .databasedumpanalyzer import DatabaseDumpAnalyzer
+from .dbconnstringanalyzer import DBConnAnalyzer
+from .emailpasswordpairanalyzer import EmailPasswordPairAnalyzer
 from .genericanalyzer import GenericAnalyzer
+from .ibananalyzer import IBANAnalyzer
 from .mailanalyzer import MailAnalyzer
+from .md5hashanalyzer import MD5HashAnalyzer
+from .microsoftkeyanalyzer import MicrosoftKeyAnalyzer
+from .originkeyanalyzer import OriginKeyAnalyzer
 from .pastebinurlanalyzer import PastebinURLAnalyzer
+from .phonenumberanalyzer import PhoneNumberAnalyzer
+from .privatekeyanalyzer import PrivateKeyAnalyzer
 from .regexanalyzer import RegexAnalyzer
+from .shahashanalyzer import SHAHashAnalyzer
+from .steamkeyanalyzer import SteamKeyAnalyzer
+from .uplaykeyanalyzer import UplayKeyAnalyzer
 from .urlanalyzer import URLAnalyzer
 from .wordanalyzer import WordAnalyzer
 from .ibananalyzer import IBANAnalyzer
+from .dbconnstringanalyzer import DBConnAnalyzer
+from .privatekeyanalyzer import PrivateKeyAnalyzer
+from .adobekeyanalyzer import AdobeKeyAnalyzer
+from .facebookaccesstokenanalyzer import FacebookAccessTokenAnalyzer
+from .base64analyzer import Base64Analyzer
+from .awssessiontokenanalyzer import AWSSessionTokenAnalyzer
 
 __all__ = (
     'AlwaysTrueAnalyzer',
@@ -26,5 +43,21 @@
     'RegexAnalyzer',
     'URLAnalyzer',
     'WordAnalyzer',
-    'IBANAnalyzer'
+    'IBANAnalyzer',
+    'DatabaseDumpAnalyzer',
+    'DBConnAnalyzer',
+    'PrivateKeyAnalyzer',
+    'PhoneNumberAnalyzer',
+    'OriginKeyAnalyzer',
+    'SteamKeyAnalyzer',
+    'UplayKeyAnalyzer',
+    'BattleNetKeyAnalyzer',
+    'MicrosoftKeyAnalyzer',
+    'AdobeKeyAnalyzer',
+    'DBConnAnalyzer',
+    'PrivateKeyAnalyzer',
+    'EmailPasswordPairAnalyzer',
+    'FacebookAccessTokenAnalyzer',
+    'Base64Analyzer',
+    'AWSSessionTokenAnalyzer'
 )

pastepwn/analyzers/adobekeyanalyzer.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+from .regexanalyzer import RegexAnalyzer
+
+
+class AdobeKeyAnalyzer(RegexAnalyzer):
+    """Analyzer to match Adobe License Keys"""
+
+    def __init__(self, action):
+        """
+        Analyzer to match Adobe License Keys
+        :param action: Single action or list of actions to be executed when a paste matches
+        """
+        # Tested Regex against https://pastebin.com/fxWBGf8t
+        regex = r"\b(?<!-)[0-9]{4}(-[0-9]{4}){5}\b(?!-)"
+
+        super().__init__(action, regex)

pastepwn/analyzers/awssessiontokenanalyzer.py

@@ -0,0 +1,12 @@
+from .regexanalyzer import RegexAnalyzer
+
+
+class AWSSessionTokenAnalyzer(RegexAnalyzer):
+    """
+    Analyzer to match AWS Session Token via regex
+    """
+    name = "AWSSessionTokenAnalyzer"
+
+    def __init__(self, actions):
+        regex = r"((\\\"|'|`)?((i)?aws)?_?((i)?session)?_?((i)?token)?(\\\"|'|`)?\\\\s{0,50}(:|=>|=)\\\\s{0,50}(\\\"|'|`)?[A-Za-z0-9/+=]{16,}(\\\"|'|`)?)"
+        super().__init__(actions, regex)

pastepwn/analyzers/base64analyzer.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+from .regexanalyzer import RegexAnalyzer
+
+
+class Base64Analyzer(RegexAnalyzer):
+    """Analyzer to match base64 encoding via regex"""
+    name = "Base64Analyzer"
+
+    def __init__(self, actions):
+        regex = r"^(?:[A-Za-z0-9+\/]{4})+(?:[A-Za-z0-9+\/]{2}==|[A-Za-z0-9+\/]{3}=)?$"
+        super().__init__(actions, regex)

pastepwn/analyzers/battlenetkeyanalyzer.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+from .regexanalyzer import RegexAnalyzer
+
+
+class BattleNetKeyAnalyzer(RegexAnalyzer):
+    """
+    Analyzer to match battle.net keys via regex
+    """
+    name = "BattleNetKeyAnalyzer"
+
+    def __init__(self, actions):
+        # battle.net activation page shows 4-4-4-4 format, but there are also codes in the 4-4-5-4-4 format which work
+        regex = r"\b(?<!-)([A-Z0-9]{4}\-[A-Z0-9]{4}\-[A-Z0-9]{4}\-[A-Z0-9]{4}|[A-Z0-9]{4}\-[A-Z0-9]{4}\-[A-Z0-9]{5}\-[A-Z0-9]{4}\-[A-Z0-9]{4})\b(?!-)"
+        super().__init__(actions, regex)

pastepwn/analyzers/databasedumpanalyzer.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+from .regexanalyzer import RegexAnalyzer
+
+
+class DatabaseDumpAnalyzer(RegexAnalyzer):
+    """Analyzer to match database dump"""
+    name = "DatabaseDumpAnalyzer"
+
+    def __init__(self, actions):
+        """
+        Analyzer to match database dump
+        :param actions: A single action or a list of actions to be executed on every paste
+        """
+        # This regex match the columns of a database
+        regex = r"\(((`\w+`|\d)(\s?)+,(\s?)+)+(`\w+`|\d)\)"
+        super().__init__(actions, regex)

pastepwn/analyzers/dbconnstringanalyzer.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+from .regexanalyzer import RegexAnalyzer
+
+
+class DBConnAnalyzer(RegexAnalyzer):
+    """Analyzer to match on email addresses via regex"""
+    name = "DBConnAnalyzer"
+
+    def __init__(self, actions):
+        # https:// should be ignored
+        regex = r"(\b(?!http(s)?\b)\w+[a-zA-Z]+://[a-zA-z0-9.:,]+)"
+        super().__init__(actions, regex)

pastepwn/analyzers/emailpasswordpairanalyzer.py

@@ -0,0 +1,12 @@
+# -*- coding: utf-8 -*-
+from .regexanalyzer import RegexAnalyzer
+
+_EMAIL_PASSWORD_REGEX = r'[\w\.\+_-]+@[\w\._-]+\.[a-zA-Z]*\:[\w\.\+\!\$\#\^&\*\(\)\{\}\[\]\_\-\@\%\=\§\\\/\'\`\´\?\<\>\;\"\:\|\,\~]+$'
+
+
+class EmailPasswordPairAnalyzer(RegexAnalyzer):
+    """Analyzer to match username:password pairs"""
+    name = "EmailPasswordPairAnalyzer"
+
+    def __init__(self, actions):
+        super().__init__(actions, _EMAIL_PASSWORD_REGEX)

pastepwn/analyzers/epickeyanalyzer.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+from .regexanalyzer import RegexAnalyzer
+
+
+class EpicKeyAnalyzer(RegexAnalyzer):
+    """Analyzer to match Epic Licensing Keys"""
+
+    def __init__(self, action):
+        """
+        Analyzer to match Epic Licensing Keys
+        :param action: Single action or list of actions to be executed when a paste matches
+        """
+        # Applied general A-Z or 0-9 based on example provided
+        # Regex can be adjusted if certain characters are not valid
+        regex = r"\b(?<!-)[A-Z0-9]{5}\-[A-Z0-9]{5}\-[A-Z0-9]{5}\-[A-Z0-9]{5}\b(?!-)"
+
+        super().__init__(action, regex)

pastepwn/analyzers/facebookaccesstokenanalyzer.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+from .regexanalyzer import RegexAnalyzer
+
+
+class FacebookAccessTokenAnalyzer(RegexAnalyzer):
+    """Analyzer to match a Facebook Access Token"""
+    name = "FacebookAccessTokenAnalyzer"
+    regex = "EAACEdEose0cBA[0-9A-Za-z]+"
+
+    def __init__(self, actions):
+        """
+        Analyzer which always matches a paste to perform actions on every paste
+        :param actions: A single action or a list of actions to be executed on every paste
+        """
+        super().__init__(actions, self.regex)

pastepwn/analyzers/md5hashanalyzer.py

@@ -7,5 +7,5 @@ class MD5HashAnalyzer(RegexAnalyzer):
     name = "MD5HashAnalyzer"
 
     def __init__(self, actions):
-        regex = r"[a-f0-9]{32}"
+        regex = r"\b(?<!-)[a-f0-9]{32}\b(?!-)"
         super().__init__(actions, regex)

pastepwn/analyzers/microsoftkeyanalyzer.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+from .regexanalyzer import RegexAnalyzer
+
+
+class MicrosoftKeyAnalyzer(RegexAnalyzer):
+    """Analyzer to match Microsoft Licensing Keys"""
+
+    def __init__(self, action):
+        """
+        Analyzer to match Microsoft Licensing Keys
+        :param action: Single action or list of actions to be executed when a paste matches
+        """
+        # Tested Regex against https://pastebin.com/r3QdpFJf
+        regex = r"\b(?<!-)[2346789BCDFGHJKMNPQRTVWXY]{5}(-[2346789BCDFGHJKMNPQRTVWXY]{5}){4}\b(?!-)"
+
+        super().__init__(action, regex)

pastepwn/analyzers/originkeyanalyzer.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+from .regexanalyzer import RegexAnalyzer
+
+
+class OriginKeyAnalyzer(RegexAnalyzer):
+    """Analyzer to match Origin Keys"""
+
+    def __init__(self, action):
+        """
+        Analyzer to match Origin Keys
+        :param action: Single action or list of actions to be executed when a paste matches
+        """
+        # Tested Regex against https://pastebin.com/vyNANvwM
+        regex = r"\b(?<!-)[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}\b(?!-)"
+
+        super().__init__(action, regex)