diff --git a/Common/TaskController.py b/Common/TaskController.py index 99897cc..8a40f41 100644 --- a/Common/TaskController.py +++ b/Common/TaskController.py @@ -43,6 +43,7 @@ def __init__(self): self.Emails = [] self.ConsumerList = [] self.HtmlList = [] + self.JsonList = [] self.Tasks = [] self.ResultsList = [] self.logger = logging.getLogger("SimplyEmail.TaskController") @@ -67,7 +68,7 @@ def __init__(self): # module.execute() # Handler for each Process that will call all the modules in the Queue - def ExecuteModule(self, Task_queue, Results_queue, Html_queue, domain, verbose=False): + def ExecuteModule(self, Task_queue, Results_queue, Html_queue, Json_queue, domain, verbose=False): while True: Task = Task_queue.get() # If the queue is empty exit this proc @@ -94,7 +95,7 @@ def ExecuteModule(self, Task_queue, Results_queue, Html_queue, domain, verbose=F # Exit a API module with out a key break # Emails will be returned as a list - Emails, HtmlResults = Module.execute() + Emails, HtmlResults, JsonResults = Module.execute() if Emails: count = len(Emails) Length = " [*] " + Module.name + \ @@ -104,6 +105,8 @@ def ExecuteModule(self, Task_queue, Results_queue, Html_queue, domain, verbose=F Results_queue.put(Email) for Email in HtmlResults: Html_queue.put(Email) + for Email in JsonResults: + Json_queue.put(Email) # Task_queue.task_done() else: Message = " [*] " + Module.name + \ @@ -198,6 +201,16 @@ def HtmlPrinter(self, HtmlFinalEmailList, Domain): # error = "[!] Error building HTML file:" + e # print helpers.color(error, warning=True) + def JsonPrinter(self, JsonFinalEmailList, FullPath, Domain): + self.logger.debug("Json Printer started") + json = helpers.JsonListToJsonObj(JsonFinalEmailList, Domain) + if json: + self.logger.debug("JSON data was returned") + with open(str(FullPath), 'w') as file: + file.write(json) + self.logger.debug("JSON wrote file: " % (FullPath)) + + def CleanResults(self, domain, scope=False): # Clean Up results, remove duplicates and enforce strict Domain results (future) # Set Timeout or you wont leave the While loop @@ -243,6 +256,23 @@ def CleanResults(self, domain, scope=False): self.logger.info("Completed cleaning results") return FinalList, HtmlFinalList + def CleanJsonResults(self, domain, scope=False): + self.logger.debug("JSON Clean Results started") + SecondList = [] + FinalList = [] + if scope: + for item in self.JsonList: + SecondList.append(item) + else: + for item in self.JsonList: + if domain.lower() in item['email'].lower(): + SecondList.append(item) + for item in SecondList: + if item not in FinalList: + FinalList.append(item) + return FinalList + + def Consumer(self, Results_queue, verbose): while True: try: @@ -265,7 +295,18 @@ def HtmlConsumer(self, Html_queue, verbose): if verbose: print e - def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=False): + def JsonConsumer(self, Json_queue, verbose): + while True: + try: + item = Json_queue.get() + if item is None: + break + self.JsonList.append(item) + except Exception as e: + if verbose: + print e + + def TaskSelector(self, domain, verbose=False, scope=False, Names=False, json="", Verify=False): # Here it will check the Queue for the next task to be completed # Using the Dynamic loaded modules we can easly select which module is up # Rather than using If statment on every task that needs to be done @@ -275,6 +316,7 @@ def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=F Task_queue = multiprocessing.Queue() Results_queue = multiprocessing.Queue() Html_queue = multiprocessing.Queue() + Json_queue = multiprocessing.Queue() # How many proc will we have, pull from config file, setting up the # config file handler @@ -295,7 +337,7 @@ def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=F for thread in range(total_proc): thread = thread procs.append(multiprocessing.Process( - target=self.ExecuteModule, args=(Task_queue, Results_queue, Html_queue, domain, verbose))) + target=self.ExecuteModule, args=(Task_queue, Results_queue, Html_queue, Json_queue, domain, verbose))) for p in procs: p.daemon = True p.start() @@ -316,6 +358,10 @@ def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=F t2 = threading.Thread(target=self.HtmlConsumer, args=(Html_queue, verbose,)) t2.daemon = True t2.start() + # Start Json Consumer + t2 = threading.Thread(target=self.JsonConsumer, args=(Json_queue, verbose,)) + t2.daemon = True + t2.start() # Enter this loop so we know when to terminate the Consumer thread # This multiprocessing.active_children() is also Joining! while True: @@ -327,25 +373,31 @@ def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=F time.sleep(1) Results_queue.put(None) Html_queue.put(None) + Json_queue.put(None) # t.join() try: + JsonFinalEmailList = self.CleanJsonResults(domain, scope) FinalEmailList, HtmlFinalEmailList = self.CleanResults( domain, scope) + except Exception as e: error = " [!] Something went wrong with parsing results:" + \ str(e) print helpers.color(error, warning=True) self.logger.critical("Something went wrong with parsing results: " + str(e)) try: - FinalCount = self.printer(FinalEmailList, domain) + if not json: + FinalCount = self.printer(FinalEmailList, domain) except Exception as e: error = " [!] Something went wrong with outputixng results:" + \ str(e) print helpers.color(error, warning=True) self.logger.critical("Something went wrong with outputixng results: " + str(e)) - Results_queue.close() try: - self.HtmlPrinter(HtmlFinalEmailList, domain) + if json: + self.JsonPrinter(JsonFinalEmailList, json, domain) + else: + self.HtmlPrinter(HtmlFinalEmailList, domain) except Exception as e: error = " [!] Something went wrong with HTML results:" + \ str(e) @@ -356,10 +408,13 @@ def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=F p.join() self.logger.debug("TaskSelector processes joined!") Task_queue.close() + Results_queue.close() + Html_queue.close() + Json_queue.close() BuiltNameCount = 0 try: # If names is True - if Names: + if Names and not json: BuiltNames = self.NameBuilder( domain, FinalEmailList, Verbose=verbose) BuiltNameCount = len(BuiltNames) @@ -385,13 +440,13 @@ def TaskSelector(self, domain, verbose=False, scope=False, Names=False, Verify=F error = " [!] Something went wrong with outputting results of Built Names:" + \ str(e) print helpers.color(error, warning=True) - - self.CompletedScreen(FinalCount, BuiltNameCount, domain) + if not json: + self.CompletedScreen(FinalCount, BuiltNameCount, domain) # This is the Test version of the multi proc above, this function # Helps with testing only one module at a time. Helping with proper # Module Dev and testing before integration - def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Verify=False): + def TestModule(self, domain, module, verbose=False, scope=False, Names=False, json='', Verify=False): self.logger.debug("Starting TaskSelector for: " + str(domain)) Config = configparser.ConfigParser() Config.read("Common/SimplyEmail.ini") @@ -400,6 +455,7 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve Task_queue = multiprocessing.JoinableQueue() Results_queue = multiprocessing.Queue() Html_queue = multiprocessing.Queue() + Json_queue = multiprocessing.Queue() for Task in self.modules: if module in Task: @@ -410,7 +466,7 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve procs = [] for thread in range(total_proc): procs.append(multiprocessing.Process( - target=self.ExecuteModule, args=(Task_queue, Results_queue, Html_queue, domain, verbose))) + target=self.ExecuteModule, args=(Task_queue, Results_queue, Html_queue, Json_queue, domain, verbose))) for p in procs: p.daemon = True p.start() @@ -431,6 +487,10 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve t2 = threading.Thread(target=self.HtmlConsumer, args=(Html_queue, verbose,)) t2.daemon = True t2.start() + # Start Json Consumer + t2 = threading.Thread(target=self.JsonConsumer, args=(Json_queue, verbose,)) + t2.daemon = True + t2.start() # Enter this loop so we know when to terminate the Consumer thread # This multiprocessing.active_children() is also Joining! while True: @@ -442,8 +502,10 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve time.sleep(1) Results_queue.put(None) Html_queue.put(None) + Json_queue.put(None) # t.join() try: + JsonFinalEmailList = self.CleanJsonResults(domain, scope) FinalEmailList, HtmlFinalEmailList = self.CleanResults( domain, scope) except Exception as e: @@ -452,15 +514,18 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve print helpers.color(error, warning=True) self.logger.critical("Something went wrong with parsing results: " + str(e)) try: - FinalCount = self.printer(FinalEmailList, domain) + if not json: + FinalCount = self.printer(FinalEmailList, domain) except Exception as e: error = " [!] Something went wrong with outputting results:" + \ str(e) print helpers.color(error, warning=True) self.logger.critical("Something went wrong with outputting results: " + str(e)) - Results_queue.close() try: - self.HtmlPrinter(HtmlFinalEmailList, domain) + if json: + self.JsonPrinter(JsonFinalEmailList, json, domain) + else: + self.HtmlPrinter(HtmlFinalEmailList, domain) except Exception as e: error = " [!] Something went wrong with HTML results:" + \ str(e) @@ -471,11 +536,14 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve for p in procs: p.join() Task_queue.close() + Results_queue.close() + Html_queue.close() + Json_queue.close() # Launches a single thread to output results BuiltNameCount = 0 try: # If names is True - if Names: + if Names and not json: BuiltNames = self.NameBuilder( domain, FinalEmailList, Verbose=verbose) BuiltNameCount = len(BuiltNames) @@ -503,8 +571,8 @@ def TestModule(self, domain, module, verbose=False, scope=False, Names=False, Ve error = " [!] Something went wrong with outputting results of Built Names:" + \ str(e) print helpers.color(error, warning=True) - - self.CompletedScreen(FinalCount, BuiltNameCount, domain) + if not json: + self.CompletedScreen(FinalCount, BuiltNameCount, domain) def NameBuilder(self, domain, emaillist, Verbose=False): ''' diff --git a/Helpers/Download.py b/Helpers/Download.py index 18ee173..2815c2a 100644 --- a/Helpers/Download.py +++ b/Helpers/Download.py @@ -23,7 +23,7 @@ def __init__(self, verbose=False): except Exception as e: print e - def download_file(self, url, filetype, maxfile=100): + def download_file(self, url, filetype, maxfile=100, verify=True): """ Downloads a file using requests, @@ -43,7 +43,7 @@ def download_file(self, url, filetype, maxfile=100): try: time.sleep(2) self.logger.debug("Download started download: " + str(url)) - r = requests.get(url, stream=True, headers=self.UserAgent) + r = requests.get(url, stream=True, headers=self.UserAgent, verify=verify) with open(local_filename, 'wb+') as f: for chunk in r.iter_content(chunk_size=1024): if chunk: @@ -124,7 +124,7 @@ def GoogleCaptchaDetection(self, RawHtml): else: return False - def requesturl(self, url, useragent, timeout=5, retrytime=3, statuscode=False, raw=False): + def requesturl(self, url, useragent, timeout=5, retrytime=3, statuscode=False, raw=False, verify=True): """ A very simple request function This is setup to handle the following parms: @@ -138,7 +138,7 @@ def requesturl(self, url, useragent, timeout=5, retrytime=3, statuscode=False, r """ rawhtml = "" try: - r = requests.get(url, headers=self.UserAgent, timeout=timeout) + r = requests.get(url, headers=self.UserAgent, timeout=timeout, verify=verify) rawhtml = r.content self.logger.debug( 'Request completed: code = ' + str(r.status_code) + ' size = ' + str(len(rawhtml)) + ' url = ' + str(url)) @@ -148,7 +148,7 @@ def requesturl(self, url, useragent, timeout=5, retrytime=3, statuscode=False, r p = ' [!] Request for url timed out, retrying: ' + url self.logger.info('Request timed out, retrying: ' + url) print helpers.color(p, firewall=True) - r = requests.get(url, headers=self.UserAgent, timeout=retrytime) + r = requests.get(url, headers=self.UserAgent, timeout=retrytime, verify=verify) rawhtml = r.content except requests.exceptions.TooManyRedirects: # fail and move on, alert user diff --git a/Helpers/Parser.py b/Helpers/Parser.py index 0778457..d49041b 100644 --- a/Helpers/Parser.py +++ b/Helpers/Parser.py @@ -6,6 +6,7 @@ import logging import string import subprocess +import time from random import randint import helpers @@ -137,6 +138,23 @@ def BuildResults(self, InputList, ModuleName): FinalOutput.append(ListItem) return FinalOutput + def BuildJson(self, InputList, ModuleName): + FinalOutput = [] + currentDate = str(time.strftime("%d/%m/%Y")) + currentTime = str(time.strftime("%H:%M:%S")) + moduleName = str(ModuleName) + for email in InputList: + obj = { + 'email': email, + 'module_name': moduleName, + 'collection_time': currentTime, + 'collection_data': currentDate, + } + FinalOutput.append(obj) + # print FinalOutput + return FinalOutput + + def extendedclean(self, modulename): self.genericClean() self.urlClean() diff --git a/Helpers/helpers.py b/Helpers/helpers.py index 2c29c09..642090d 100644 --- a/Helpers/helpers.py +++ b/Helpers/helpers.py @@ -5,10 +5,52 @@ import logging import time import magic +<<<<<<< HEAD +import json +======= +>>>>>>> master import configparser import collections from fake_useragent import UserAgent +<<<<<<< HEAD +def dictToJson(inputDict): + """ + Takes in a list of dict items. + Converts them to json and returns list of json obj. + """ + obj = [] + for item in inputDict: + obj += json.dumps(item) + return obj + +def JsonListToJsonObj(inputJsonList, domain): + """ + Takes a list of json objects, + places them in a key and returns the data. + """ + currentDate = str(time.strftime("%d/%m/%Y")) + currentTime = str(time.strftime("%H:%M:%S")) + currentTool = "SimplyEmail" + config = configparser.ConfigParser() + config.read('Common/SimplyEmail.ini') + currentVersion = str(config['GlobalSettings']['Version']) + count = len(inputJsonList) + dic = collections.OrderedDict() + dic = { + "domain_of_collection" : domain, + "data_of_collection" : currentDate, + "time_of_collection" : currentTime, + "tool_of_collection" : currentTool, + "current_version" : currentVersion, + "email_collection_count" : count, + "emails" : inputJsonList, + } + obj = json.dumps(dic, indent=4, sort_keys=True) + return obj + +======= +>>>>>>> master def color(string, status=True, warning=False, bold=True, blue=False, firewall=False): # Change text color for the linux terminal, defaults to green. # Set "warning=True" for red. diff --git a/Modules/AskSearch.py b/Modules/AskSearch.py index 9db30fc..f064494 100644 --- a/Modules/AskSearch.py +++ b/Modules/AskSearch.py @@ -39,8 +39,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.logger.debug("AskSearch module started") self.process() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def process(self): dl = Download.Download(self.verbose) @@ -70,5 +70,6 @@ def process(self): def get_emails(self): parse = Parser.Parser(self.Html) FinalOutput, HtmlResults = parse.extendedclean(self.name) + JsonResults = parse.BuildJson(FinalOutput, self.name) self.logger.debug('AskSearch completed search') - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/CanarioAPI.py b/Modules/CanarioAPI.py index f83c4fd..14f9bfa 100644 --- a/Modules/CanarioAPI.py +++ b/Modules/CanarioAPI.py @@ -45,8 +45,8 @@ def __init__(self, domain, verbose=False): def execute(self): self.process() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def process(self): try: @@ -119,4 +119,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/CanaryBinSearch.py b/Modules/CanaryBinSearch.py index df8dad3..ef8fff5 100644 --- a/Modules/CanaryBinSearch.py +++ b/Modules/CanaryBinSearch.py @@ -54,7 +54,7 @@ def execute(self): self.logger.debug("CanaryBinSearch module started") self.process() FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults def process(self): # Get all the Pastebin raw items @@ -71,7 +71,7 @@ def process(self): url = "https://canary.pw/search/?q=" + str(self.domain) + "&page=" + \ str(self.Counter) rawhtml, statuscode = dl.requesturl( - url, useragent=self.UserAgent, statuscode=True) + url, useragent=self.UserAgent, statuscode=True, verify=False) if statuscode != 200: break except Exception as e: @@ -117,5 +117,6 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) + JsonResults = Parse.BuildJson(FinalOutput, self.name) self.logger.debug('CanaryBinSearch completed search') - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/EmailHunter.py b/Modules/EmailHunter.py index 63420d8..d09df1e 100644 --- a/Modules/EmailHunter.py +++ b/Modules/EmailHunter.py @@ -35,8 +35,8 @@ def __init__(self, domain, verbose=False): def execute(self): self.logger.debug("EmailHunter module started") self.process() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def process(self): dl = Download.Download(self.verbose) @@ -77,5 +77,6 @@ def get_emails(self): Parse = Parser.Parser(self.results) FinalOutput = Parse.CleanListOutput() HtmlResults = Parse.BuildResults(FinalOutput, self.name) + JsonResults = Parse.BuildJson(FinalOutput, self.name) self.logger.debug('EmailHunter completed search') - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/ExaleadDOCSearch.py b/Modules/ExaleadDOCSearch.py index 924a475..654bfb0 100644 --- a/Modules/ExaleadDOCSearch.py +++ b/Modules/ExaleadDOCSearch.py @@ -47,8 +47,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.logger.debug("ExaleadDOCSearch module started") self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): dl = Download.Download(self.verbose) @@ -124,5 +124,6 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) + JsonResults = Parse.BuildJson(FinalOutput, self.name) self.logger.debug('ExaleadDOCSearch completed search') - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/ExaleadDOCXSearch.py b/Modules/ExaleadDOCXSearch.py index 9e692b8..ed42866 100644 --- a/Modules/ExaleadDOCXSearch.py +++ b/Modules/ExaleadDOCXSearch.py @@ -49,8 +49,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.logger.debug("ExaleadDOCXSearch module started") self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def download_file(self, url): local_filename = url.split('/')[-1] @@ -142,5 +142,6 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) + JsonResults = Parse.BuildJson(FinalOutput, self.name) self.logger.debug('ExaleadDOCXSearch completed search') - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/ExaleadPDFSearch.py b/Modules/ExaleadPDFSearch.py index 8ca2f07..2a412ac 100644 --- a/Modules/ExaleadPDFSearch.py +++ b/Modules/ExaleadPDFSearch.py @@ -38,8 +38,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): convert = Converter.Converter(verbose=self.verbose) @@ -109,4 +109,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/ExaleadPPTXSearch.py b/Modules/ExaleadPPTXSearch.py index 45f49c8..15f0304 100644 --- a/Modules/ExaleadPPTXSearch.py +++ b/Modules/ExaleadPPTXSearch.py @@ -46,8 +46,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.logger.debug("ExaleadPPTXSearch module started") self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): dl = Download.Download(self.verbose) @@ -123,5 +123,6 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) + JsonResults = Parse.BuildJson(FinalOutput, self.name) self.logger.debug('ExaleadPPTXSearch completed search') - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/ExaleadSearch.py b/Modules/ExaleadSearch.py index e8a903f..a3a8e41 100644 --- a/Modules/ExaleadSearch.py +++ b/Modules/ExaleadSearch.py @@ -36,8 +36,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): while self.Counter <= self.Limit: @@ -89,4 +89,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/ExaleadXLSXSearch.py b/Modules/ExaleadXLSXSearch.py index d6f6068..dda950c 100644 --- a/Modules/ExaleadXLSXSearch.py +++ b/Modules/ExaleadXLSXSearch.py @@ -44,8 +44,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): dl = Download.Download(verbose=self.verbose) @@ -116,4 +116,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/FlickrSearch.py b/Modules/FlickrSearch.py index 9314206..25c1d19 100644 --- a/Modules/FlickrSearch.py +++ b/Modules/FlickrSearch.py @@ -31,8 +31,8 @@ def __init__(self, domain, verbose=False): def execute(self): self.process() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def process(self): dl = Download.Download(verbose=self.verbose) @@ -53,4 +53,5 @@ def get_emails(self): Parse = Parser.Parser(self.results) FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/GitHubCodeSearch.py b/Modules/GitHubCodeSearch.py index bd7c1bc..9593461 100644 --- a/Modules/GitHubCodeSearch.py +++ b/Modules/GitHubCodeSearch.py @@ -59,8 +59,8 @@ def __init__(self, domain, verbose=False): def execute(self): self.process() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def process(self): dl = Download.Download(verbose=self.verbose) @@ -105,4 +105,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/GitHubGistSearch.py b/Modules/GitHubGistSearch.py index 87a1b78..dc4538a 100644 --- a/Modules/GitHubGistSearch.py +++ b/Modules/GitHubGistSearch.py @@ -36,8 +36,8 @@ def __init__(self, domain, verbose=False): def execute(self): self.process() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def process(self): dl = Download.Download(verbose=self.verbose) @@ -85,4 +85,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/GitHubUserSearch.py b/Modules/GitHubUserSearch.py index 426968e..03fbe03 100644 --- a/Modules/GitHubUserSearch.py +++ b/Modules/GitHubUserSearch.py @@ -33,8 +33,8 @@ def __init__(self, domain, verbose=False): def execute(self): self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): dl = Download.Download(verbose=self.verbose) @@ -66,4 +66,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/GoogleCsvSearch.py b/Modules/GoogleCsvSearch.py index 8c59cfd..783583f 100644 --- a/Modules/GoogleCsvSearch.py +++ b/Modules/GoogleCsvSearch.py @@ -37,8 +37,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): dl = Download.Download(self.verbose) @@ -112,4 +112,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/GoogleDocSearch.py b/Modules/GoogleDocSearch.py index 4b1f19d..1028476 100644 --- a/Modules/GoogleDocSearch.py +++ b/Modules/GoogleDocSearch.py @@ -39,8 +39,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): dl = Download.Download(self.verbose) @@ -115,4 +115,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/GoogleDocxSearch.py b/Modules/GoogleDocxSearch.py index f3e51d4..a393930 100644 --- a/Modules/GoogleDocxSearch.py +++ b/Modules/GoogleDocxSearch.py @@ -40,8 +40,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): dl = Download.Download(self.verbose) @@ -112,4 +112,5 @@ def get_emails(self): # Unicode issues here: FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/GooglePDFSearch.py b/Modules/GooglePDFSearch.py index 7425337..d9d4571 100644 --- a/Modules/GooglePDFSearch.py +++ b/Modules/GooglePDFSearch.py @@ -39,8 +39,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): # setup for helpers in the download class @@ -114,4 +114,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/GooglePPTXSearch.py b/Modules/GooglePPTXSearch.py index 265d43b..ca0bbb6 100644 --- a/Modules/GooglePPTXSearch.py +++ b/Modules/GooglePPTXSearch.py @@ -38,8 +38,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): convert = Converter.Converter(self.verbose) @@ -124,4 +124,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/GoogleSearch.py b/Modules/GoogleSearch.py index 9f3e238..0b18910 100644 --- a/Modules/GoogleSearch.py +++ b/Modules/GoogleSearch.py @@ -34,8 +34,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): dl = Download.Download(self.verbose) @@ -71,4 +71,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/GoogleXLSXSearch.py b/Modules/GoogleXLSXSearch.py index 4807b46..b1eed64 100644 --- a/Modules/GoogleXLSXSearch.py +++ b/Modules/GoogleXLSXSearch.py @@ -44,8 +44,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.logger.debug("GoogleXlsxSearch Started") self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): convert = Converter.Converter(verbose=self.verbose) @@ -129,5 +129,6 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) + JsonResults = Parse.BuildJson(FinalOutput, self.name) self.logger.debug('GoogleXlsxSearch completed search') - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/HtmlScrape.py b/Modules/HtmlScrape.py index a1b08ce..67b9cef 100644 --- a/Modules/HtmlScrape.py +++ b/Modules/HtmlScrape.py @@ -47,8 +47,8 @@ def __init__(self, domain, verbose=False): def execute(self): try: self.search() - Emails, HtmlResults = self.get_emails() - return Emails, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults except Exception as e: print e @@ -114,6 +114,11 @@ def get_emails(self): if self.remove == "yes" or self.remove == "Yes": if not self.retVal > 0: shutil.rmtree(directory) + try: + shutil.rmtree(directory) + except: + pass Parse = Parser.Parser(FinalOutput) HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/OnInstagram.py b/Modules/OnInstagram.py index 75e8338..f248347 100644 --- a/Modules/OnInstagram.py +++ b/Modules/OnInstagram.py @@ -32,8 +32,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.process() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def process(self): try: @@ -55,4 +55,5 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) - return FinalOutput, HtmlResults + JsonResults = Parse.BuildJson(FinalOutput, self.name) + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/PasteBinSearch.py b/Modules/PasteBinSearch.py index c9c9fa4..0441aa0 100644 --- a/Modules/PasteBinSearch.py +++ b/Modules/PasteBinSearch.py @@ -42,8 +42,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.logger.debug("PasteBinSearch started") self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): dl = Download.Download(self.verbose) @@ -115,5 +115,6 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) + JsonResults = Parse.BuildJson(FinalOutput, self.name) self.logger.debug("PasteBinSearch completed search") - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/RedditPostSearch.py b/Modules/RedditPostSearch.py index cc133c7..ff6979b 100644 --- a/Modules/RedditPostSearch.py +++ b/Modules/RedditPostSearch.py @@ -38,8 +38,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.logger.debug("RedditPostSearch started") self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): dl = Download.Download(self.verbose) @@ -77,5 +77,6 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) + JsonResults = Parse.BuildJson(FinalOutput, self.name) self.logger.debug("RedditPostSearch completed search") - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/SearchPGP.py b/Modules/SearchPGP.py index 00ba007..0bf5e1b 100644 --- a/Modules/SearchPGP.py +++ b/Modules/SearchPGP.py @@ -36,8 +36,8 @@ def __init__(self, domain, verbose=False): def execute(self): self.logger.debug("SearchPGP started") self.process() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def process(self): try: @@ -59,5 +59,6 @@ def get_emails(self): Parse = Parser.Parser(self.results) FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) + JsonResults = Parse.BuildJson(FinalOutput, self.name) self.logger.debug("SearchPGP completed search") - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/WhoisAPISearch.py b/Modules/WhoisAPISearch.py index c68c01e..03ec703 100644 --- a/Modules/WhoisAPISearch.py +++ b/Modules/WhoisAPISearch.py @@ -36,8 +36,8 @@ def __init__(self, domain, verbose=False): def execute(self): self.logger.debug("WhoisAPISearch Started") self.process() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def process(self): try: @@ -59,5 +59,6 @@ def get_emails(self): Parse = Parser.Parser(self.results) FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) + JsonResults = Parse.BuildJson(FinalOutput, self.name) self.logger.debug('WhoisAPISearch completed search') - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/Whoisolgy.py b/Modules/Whoisolgy.py index 28c4a2e..25efae9 100644 --- a/Modules/Whoisolgy.py +++ b/Modules/Whoisolgy.py @@ -37,8 +37,8 @@ def __init__(self, domain, verbose=False): def execute(self): self.logger.debug("Whoisology Started") self.process() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def process(self): try: @@ -61,5 +61,6 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) + JsonResults = Parse.BuildJson(FinalOutput, self.name) self.logger.debug('Whoisology completed search') - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults diff --git a/Modules/YahooSearch.py b/Modules/YahooSearch.py index 8593d0d..52f359f 100644 --- a/Modules/YahooSearch.py +++ b/Modules/YahooSearch.py @@ -44,8 +44,8 @@ def __init__(self, Domain, verbose=False): def execute(self): self.logger.debug("AskSearch Started") self.search() - FinalOutput, HtmlResults = self.get_emails() - return FinalOutput, HtmlResults + FinalOutput, HtmlResults, JsonResults = self.get_emails() + return FinalOutput, HtmlResults, JsonResults def search(self): while self.Counter <= self.Limit and self.Counter <= 1000: @@ -79,5 +79,6 @@ def get_emails(self): Parse.urlClean() FinalOutput = Parse.GrepFindEmails() HtmlResults = Parse.BuildResults(FinalOutput, self.name) + JsonResults = Parse.BuildJson(FinalOutput, self.name) self.logger.debug('YahooSearch completed search') - return FinalOutput, HtmlResults + return FinalOutput, HtmlResults, JsonResults \ No newline at end of file diff --git a/README.md b/README.md index 6f07780..75bd809 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,9 @@ or with email verification or with email verification & Name Creation ./SimplyEmail.py -all -v -verify -n -e cybersyndicates.com + +or json automation +./SimplyEmail.py -all -e cybersyndicates.com --json cs-json.txt ``` This will run ALL modules that are have API Key placed in the SimpleEmail.ini file and will run all non-API based modules. @@ -272,6 +275,33 @@ sc@mandiant.com je@mandiant.com su@mandiant.com ``` +### JSON Output +using the ```--json test.txt``` flag will alow you to output standard JSON text file for automation needs. This can be currently used with the email scraping portion only, maybe name generation and email verification to come. These helpers will be soon in the SQL DB and API for more streamline automation. Example output: +``` +{ + "current_version": "v1.4.1", + "data_of_collection": "26/06/2016", + "domain_of_collection": "---SNIP---", + "email_collection_count": 220, + "emails": [ + { + "collection_data": "26/06/2016", + "collection_time": "18:47:42", + "email": "---SNIP---", + "module_name": "Searching PGP" + }, + ---SNIP--- + { + "collection_data": "26/06/2016", + "collection_time": "18:51:46", + "email": "---SNIP---", + "module_name": "Exalead PDF Search for Emails" + } + ], + "time_of_collection": "18:53:04", + "tool_of_collection": "SimplyEmail" +} +``` ### HTML Output: As I mentioned before a powerful function that I wanted to integrate was the ability to produce a visually appealing and rich report for the user and potentially something that could be part of data provided to a client. Please let me know with suggestions! #### Email Source: diff --git a/SimplyEmail.py b/SimplyEmail.py index c63bb81..bdc4df0 100755 --- a/SimplyEmail.py +++ b/SimplyEmail.py @@ -38,13 +38,16 @@ def cli_parser(): "-verify", action='store_true', help="Set this to enable SMTP server email verify") parser.add_argument( "-v", action='store_true', help="Set this switch for verbose output of modules") + parser.add_argument( + "--json", metavar='json-emails.txt', default="", + help="Set this switch for json output to specfic file") parser.add_argument('-h', '-?', '--h', '-help', '--help', action="store_true", help=argparse.SUPPRESS) args = parser.parse_args() if args.h: parser.print_help() sys.exit() - return args.all, args.e, args.l, args.t, args.s, args.n, args.verify, args.v + return args.all, args.e, args.l, args.t, args.s, args.n, args.verify, args.v, args.json def TaskStarter(version): @@ -53,7 +56,7 @@ def TaskStarter(version): # need to pass the store true somehow to tell printer to restrict output log = helpers.log() log.start() - cli_all, cli_domain, cli_list, cli_test, cli_scope, cli_names, cli_verify, cli_verbose = cli_parser() + cli_all, cli_domain, cli_list, cli_test, cli_scope, cli_names, cli_verify, cli_verbose, cli_json = cli_parser() cli_domain = cli_domain.lower() Task = TaskController.Conducter() Task.load_modules() @@ -73,13 +76,15 @@ def TaskStarter(version): V = VersionCheck.VersionCheck(version) V.VersionRequest() Task.TestModule(cli_domain, cli_test, verbose=cli_verbose, - scope=cli_scope, Names=cli_names, Verify=cli_verify) + scope=cli_scope, Names=cli_names, Verify=cli_verify, + json=cli_json) if cli_all: log.infomsg("Tasked to run all Modules on domain: " + cli_domain, "Main") V = VersionCheck.VersionCheck(version) V.VersionRequest() Task.TaskSelector(cli_domain, verbose=cli_verbose, - scope=cli_scope, Names=cli_names, Verify=cli_verify) + scope=cli_scope, Names=cli_names, Verify=cli_verify, + json=cli_json) # def GenerateReport(): diff --git a/tests/test_simplyemail_list.py b/tests/test_simplyemail_list.py index 6469201..1d7f718 100644 --- a/tests/test_simplyemail_list.py +++ b/tests/test_simplyemail_list.py @@ -60,33 +60,33 @@ def test_taskcontrollers(): def test_searchpgp(): s = SearchPGP.ClassName('verisgroup.com', verbose=True) - FinalOutput, HtmlResults = s.execute() + FinalOutput, HtmlResults, JsonResults = s.execute() assert 'jmacovei@verisgroup.com' in FinalOutput def test_asksearch(): s = AskSearch.ClassName('gmail.com', verbose=True) - FinalOutput, HtmlResults = s.execute() + FinalOutput, HtmlResults, JsonResults = s.execute() assert len(FinalOutput) > 0 def test_yahoosearch(): s = YahooSearch.ClassName('gmail.com', verbose=True) - FinalOutput, HtmlResults = s.execute() + FinalOutput, HtmlResults, JsonResults = s.execute() assert len(FinalOutput) > 0 def test_whoisapi(): s = WhoisAPISearch.ClassName('verisgroup.com', verbose=True) - FinalOutput, HtmlResults = s.execute() + FinalOutput, HtmlResults, JsonResults = s.execute() assert 'abuse@web.com' in FinalOutput def test_redditsearch(): s = RedditPostSearch.ClassName('gmail.com', verbose=True) - FinalOutput, HtmlResults = s.execute() + FinalOutput, HtmlResults, JsonResults = s.execute() #assert '@gmail.com' in FinalOutput #Look into this issue def test_flickrsearch(): s = FlickrSearch.ClassName('microsoft.com', verbose=True) - FinalOutput, HtmlResults = s.execute() + FinalOutput, HtmlResults, JsonResults = s.execute() def test_downloads(): # perfrom Download testing