Merge pull request #20 from killswitch-GUI/dev-json

json support

Common/TaskController.py

@@ -43,6 +43,7 @@ def __init__(self):
         self.Emails = []
         self.ConsumerList = []
         self.HtmlList = []
+        self.JsonList = []
         self.Tasks = []
         self.ResultsList = []
         self.logger = logging.getLogger("SimplyEmail.TaskController")
@@ -67,7 +68,7 @@ def __init__(self):
     #     module.execute()
 
     # Handler for each Process that will call all the modules in the Queue
-    def ExecuteModule(self, Task_queue, Results_queue, Html_queue, domain, verbose=False):
+    def ExecuteModule(self, Task_queue, Results_queue, Html_queue, Json_queue, domain, verbose=False):
         while True:
             Task = Task_queue.get()
             # If the queue is empty exit this proc
@@ -94,7 +95,7 @@ def ExecuteModule(self, Task_queue, Results_queue, Html_queue, domain, verbose=F
                             # Exit a API module with out a key
                             break
                     # Emails will be returned as a list
-                    Emails, HtmlResults = Module.execute()
+                    Emails, HtmlResults, JsonResults = Module.execute()
                     if Emails:
                         count = len(Emails)
                         Length = " [*] " + Module.name + \
@@ -104,6 +105,8 @@ def ExecuteModule(self, Task_queue, Results_queue, Html_queue, domain, verbose=F
                             Results_queue.put(Email)
                         for Email in HtmlResults:
                             Html_queue.put(Email)
+                        for Email in JsonResults:
+                            Json_queue.put(Email)
                         # Task_queue.task_done()
                     else:
                         Message = " [*] " + Module.name + \
@@ -198,6 +201,16 @@ def HtmlPrinter(self, HtmlFinalEmailList, Domain):
         # error =  "[!] Error building HTML file:" + e
         # print helpers.color(error, warning=True)
 
+    def JsonPrinter(self, JsonFinalEmailList, FullPath, Domain):
+        self.logger.debug("Json Printer started")
+        json = helpers.JsonListToJsonObj(JsonFinalEmailList, Domain)
+        if json:
+            self.logger.debug("JSON data was returned")
+            with open(str(FullPath), 'w') as file:
+                file.write(json)
+            self.logger.debug("JSON wrote file: " % (FullPath))
+
+
     def CleanResults(self, domain, scope=False):
         # Clean Up results, remove duplicates and enforce strict Domain results (future)
         # Set Timeout or you wont leave the While loop
@@ -243,6 +256,23 @@ def CleanResults(self, domain, scope=False):
         self.logger.info("Completed cleaning results")
         return FinalList, HtmlFinalList
 
+    def CleanJsonResults(self, domain, scope=False):
+        self.logger.debug("JSON Clean Results started")
+        SecondList = []
+        FinalList = []
+        if scope:
+            for item in self.JsonList:
+                SecondList.append(item)
+        else:
+            for item in self.JsonList:
+                if domain.lower() in item['email'].lower():
+                    SecondList.append(item)
+        for item in SecondList:
+            if item not in FinalList:
+                FinalList.append(item)
+        return FinalList
+
+
     def Consumer(self, Results_queue, verbose):
         while True:
             try:
@@ -265,7 +295,18 @@ def HtmlConsumer(self, Html_queue, verbose):
                 if verbose:
                     print e
 
-    def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=False):
+    def JsonConsumer(self, Json_queue, verbose):
+        while True:
+            try:
+                item = Json_queue.get()
+                if item is None:
+                    break
+                self.JsonList.append(item)
+            except Exception as e:
+                if verbose:
+                    print e
+
+    def TaskSelector(self, domain, verbose=False, scope=False, Names=False, json="", Verify=False):
         # Here it will check the Queue for the next task to be completed
         # Using the Dynamic loaded modules we can easly select which module is up
         # Rather than using If statment on every task that needs to be done
@@ -275,6 +316,7 @@ def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=F
         Task_queue = multiprocessing.Queue()
         Results_queue = multiprocessing.Queue()
         Html_queue = multiprocessing.Queue()
+        Json_queue = multiprocessing.Queue()
 
         # How many proc will we have, pull from config file, setting up the
         # config file handler
@@ -295,7 +337,7 @@ def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=F
         for thread in range(total_proc):
             thread = thread
             procs.append(multiprocessing.Process(
-                target=self.ExecuteModule, args=(Task_queue, Results_queue, Html_queue, domain, verbose)))
+                target=self.ExecuteModule, args=(Task_queue, Results_queue, Html_queue, Json_queue, domain, verbose)))
         for p in procs:
             p.daemon = True
             p.start()
@@ -316,6 +358,10 @@ def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=F
         t2 = threading.Thread(target=self.HtmlConsumer, args=(Html_queue, verbose,))
         t2.daemon = True
         t2.start()
+        # Start Json Consumer
+        t2 = threading.Thread(target=self.JsonConsumer, args=(Json_queue, verbose,))
+        t2.daemon = True
+        t2.start()
         # Enter this loop so we know when to terminate the Consumer thread
         # This multiprocessing.active_children() is also Joining!
         while True:
@@ -327,25 +373,31 @@ def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=F
                 time.sleep(1)
                 Results_queue.put(None)
                 Html_queue.put(None)
+                Json_queue.put(None)
                 # t.join()
                 try:
+                    JsonFinalEmailList = self.CleanJsonResults(domain, scope)
                     FinalEmailList, HtmlFinalEmailList = self.CleanResults(
                         domain, scope)
+
                 except Exception as e:
                     error = " [!] Something went wrong with parsing results:" + \
                         str(e)
                     print helpers.color(error, warning=True)
                     self.logger.critical("Something went wrong with parsing results: " + str(e))
                 try:
-                    FinalCount = self.printer(FinalEmailList, domain)
+                    if not json:
+                        FinalCount = self.printer(FinalEmailList, domain)
                 except Exception as e:
                     error = " [!] Something went wrong with outputixng results:" + \
                         str(e)
                     print helpers.color(error, warning=True)
                     self.logger.critical("Something went wrong with outputixng results: " + str(e))
-                Results_queue.close()
                 try:
-                    self.HtmlPrinter(HtmlFinalEmailList, domain)
+                    if json:
+                        self.JsonPrinter(JsonFinalEmailList, json, domain)
+                    else:
+                        self.HtmlPrinter(HtmlFinalEmailList, domain)
                 except Exception as e:
                     error = " [!] Something went wrong with HTML results:" + \
                         str(e)
@@ -356,10 +408,13 @@ def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=F
             p.join()
             self.logger.debug("TaskSelector processes joined!")
         Task_queue.close()
+        Results_queue.close()
+        Html_queue.close()
+        Json_queue.close()
         BuiltNameCount = 0
         try:
             # If names is True
-            if Names:
+            if Names and not json:
                 BuiltNames = self.NameBuilder(
                     domain, FinalEmailList, Verbose=verbose)
                 BuiltNameCount = len(BuiltNames)
@@ -385,13 +440,13 @@ def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=F
             error = " [!] Something went wrong with outputting results of Built Names:" + \
                 str(e)
             print helpers.color(error, warning=True)
-
-        self.CompletedScreen(FinalCount, BuiltNameCount, domain)
+        if not json:
+            self.CompletedScreen(FinalCount, BuiltNameCount, domain)
 
     # This is the Test version of the multi proc above, this function
     # Helps with testing only one module at a time. Helping with proper
     # Module Dev and testing before integration
-    def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Verify=False):
+    def TestModule(self, domain, module, verbose=False, scope=False, Names=False, json='', Verify=False):
         self.logger.debug("Starting TaskSelector for: " + str(domain))
         Config = configparser.ConfigParser()
         Config.read("Common/SimplyEmail.ini")
@@ -400,6 +455,7 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve
         Task_queue = multiprocessing.JoinableQueue()
         Results_queue = multiprocessing.Queue()
         Html_queue = multiprocessing.Queue()
+        Json_queue = multiprocessing.Queue()
 
         for Task in self.modules:
             if module in Task:
@@ -410,7 +466,7 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve
         procs = []
         for thread in range(total_proc):
             procs.append(multiprocessing.Process(
-                target=self.ExecuteModule, args=(Task_queue, Results_queue, Html_queue, domain, verbose)))
+                target=self.ExecuteModule, args=(Task_queue, Results_queue, Html_queue, Json_queue, domain, verbose)))
         for p in procs:
             p.daemon = True
             p.start()
@@ -431,6 +487,10 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve
         t2 = threading.Thread(target=self.HtmlConsumer, args=(Html_queue, verbose,))
         t2.daemon = True
         t2.start()
+        # Start Json Consumer
+        t2 = threading.Thread(target=self.JsonConsumer, args=(Json_queue, verbose,))
+        t2.daemon = True
+        t2.start()
         # Enter this loop so we know when to terminate the Consumer thread
         # This multiprocessing.active_children() is also Joining!
         while True:
@@ -442,8 +502,10 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve
                 time.sleep(1)
                 Results_queue.put(None)
                 Html_queue.put(None)
+                Json_queue.put(None)
                 # t.join()
                 try:
+                    JsonFinalEmailList = self.CleanJsonResults(domain, scope)
                     FinalEmailList, HtmlFinalEmailList = self.CleanResults(
                         domain, scope)
                 except Exception as e:
@@ -452,15 +514,18 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve
                     print helpers.color(error, warning=True)
                     self.logger.critical("Something went wrong with parsing results: " + str(e))
                 try:
-                    FinalCount = self.printer(FinalEmailList, domain)
+                    if not json:
+                        FinalCount = self.printer(FinalEmailList, domain)
                 except Exception as e:
                     error = " [!] Something went wrong with outputting results:" + \
                         str(e)
                     print helpers.color(error, warning=True)
                     self.logger.critical("Something went wrong with outputting results: " + str(e))
-                Results_queue.close()
                 try:
-                    self.HtmlPrinter(HtmlFinalEmailList, domain)
+                    if json:
+                        self.JsonPrinter(JsonFinalEmailList, json, domain)
+                    else:
+                        self.HtmlPrinter(HtmlFinalEmailList, domain)
                 except Exception as e:
                     error = " [!] Something went wrong with HTML results:" + \
                         str(e)
@@ -471,11 +536,14 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve
         for p in procs:
             p.join()
         Task_queue.close()
+        Results_queue.close()
+        Html_queue.close()
+        Json_queue.close()
         # Launches a single thread to output results
         BuiltNameCount = 0
         try:
             # If names is True
-            if Names:
+            if Names and not json:
                 BuiltNames = self.NameBuilder(
                     domain, FinalEmailList, Verbose=verbose)
                 BuiltNameCount = len(BuiltNames)
@@ -503,8 +571,8 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve
             error = " [!] Something went wrong with outputting results of Built Names:" + \
                 str(e)
             print helpers.color(error, warning=True)
-
-        self.CompletedScreen(FinalCount, BuiltNameCount, domain)
+        if not json:
+            self.CompletedScreen(FinalCount, BuiltNameCount, domain)
 
     def NameBuilder(self, domain, emaillist, Verbose=False):
         '''

Helpers/Download.py

@@ -23,7 +23,7 @@ def __init__(self, verbose=False):
         except Exception as e:
             print e
 
-    def download_file(self, url, filetype, maxfile=100):
+    def download_file(self, url, filetype, maxfile=100, verify=True):
         """
         Downloads a file using requests,
 
@@ -43,7 +43,7 @@ def download_file(self, url, filetype, maxfile=100):
         try:
             time.sleep(2)
             self.logger.debug("Download started download: " + str(url))
-            r = requests.get(url, stream=True, headers=self.UserAgent)
+            r = requests.get(url, stream=True, headers=self.UserAgent, verify=verify)
             with open(local_filename, 'wb+') as f:
                 for chunk in r.iter_content(chunk_size=1024):
                     if chunk:
@@ -124,7 +124,7 @@ def GoogleCaptchaDetection(self, RawHtml):
         else:
             return False
 
-    def requesturl(self, url, useragent, timeout=5, retrytime=3, statuscode=False, raw=False):
+    def requesturl(self, url, useragent, timeout=5, retrytime=3, statuscode=False, raw=False, verify=True):
         """
         A very simple request function
         This is setup to handle the following parms:
@@ -138,7 +138,7 @@ def requesturl(self, url, useragent, timeout=5, retrytime=3, statuscode=False, r
         """
         rawhtml = ""
         try:
-            r = requests.get(url, headers=self.UserAgent, timeout=timeout)
+            r = requests.get(url, headers=self.UserAgent, timeout=timeout, verify=verify)
             rawhtml = r.content
             self.logger.debug(
                 'Request completed: code = ' + str(r.status_code) + ' size = ' + str(len(rawhtml)) + ' url = ' + str(url))
@@ -148,7 +148,7 @@ def requesturl(self, url, useragent, timeout=5, retrytime=3, statuscode=False, r
                 p = ' [!] Request for url timed out, retrying: ' + url
                 self.logger.info('Request timed out, retrying: ' + url)
                 print helpers.color(p, firewall=True)
-            r = requests.get(url, headers=self.UserAgent, timeout=retrytime)
+            r = requests.get(url, headers=self.UserAgent, timeout=retrytime, verify=verify)
             rawhtml = r.content
         except requests.exceptions.TooManyRedirects:
             # fail and move on, alert user

Helpers/Parser.py

@@ -6,6 +6,7 @@
 import logging
 import string
 import subprocess
+import time
 from random import randint
 import helpers
 
@@ -137,6 +138,23 @@ def BuildResults(self, InputList, ModuleName):
             FinalOutput.append(ListItem)
         return FinalOutput
 
+    def BuildJson(self, InputList, ModuleName):
+        FinalOutput = []
+        currentDate = str(time.strftime("%d/%m/%Y"))
+        currentTime = str(time.strftime("%H:%M:%S"))
+        moduleName = str(ModuleName)
+        for email in InputList:
+            obj = {
+                'email': email,
+                'module_name': moduleName,
+                'collection_time': currentTime,
+                'collection_data': currentDate,
+            }
+            FinalOutput.append(obj)
+        # print FinalOutput
+        return FinalOutput
+
+
     def extendedclean(self, modulename):
         self.genericClean()
         self.urlClean()