Skip to content
Permalink
Browse files

Updated the profiler sites file. (@WebBreacher)

Added a comment to the metacrawler module.

Updated the LICENSE file.

Changed the default options for the unmangle module.

Updated the hashes_org module to leverage the new API.

Updated the metacrawler module to handle errors more gracefully.

Updated the ipinfodb module to account for rate limiting.

Fixed a bug in the fullcontact module and modified the module to account for rate limiting.

Updated the core base module to account for hashes_org API changes.

Fixed a parsing bug in the search mixin.

Updated the parsers util to extract all metadata keys.

Added the migrate_hosts module.

Added the ssltools module. (@tmaletic)

Removed unecessary ORDER BY clauses from module default queries.

Added the censysio module. (@skew)

Resurrected the youtube pushpin module with a new API.

Added the shodan_ip module. (@T3LC0)

Modified the brute_hosts modules to resolve hosts through a wildcard record.

Added show command aliases for workspaces and keys.

Updated the GHDB.
  • Loading branch information...
lanmaster53 committed Sep 23, 2015
1 parent 131c609 commit 3a211b7bc495c862967b483711839169869d42ea
Showing with 775 additions and 151 deletions.
  1. +3 −3 LICENSE
  2. +1 −1 VERSION
  3. +400 −0 data/ghdb.json
  4. +45 −37 data/profiler_sites.json
  5. +1 −1 modules/discovery/info_disclosure/interesting_files.py
  6. +1 −1 modules/recon/companies-contacts/jigsaw/search_contacts.py
  7. +1 −1 modules/recon/companies-contacts/jigsaw_auth.py
  8. +1 −1 modules/recon/companies-contacts/linkedin_auth.py
  9. +1 −1 modules/recon/contacts-contacts/unmangle.py
  10. +1 −1 modules/recon/contacts-credentials/hibp_breach.py
  11. +1 −1 modules/recon/contacts-credentials/hibp_paste.py
  12. +1 −1 modules/recon/contacts-credentials/pwnedlist.py
  13. +7 −1 modules/recon/contacts-profiles/fullcontact.py
  14. +15 −51 modules/recon/credentials-credentials/hashes_org.py
  15. +18 −11 modules/recon/domains-contacts/metacrawler.py
  16. +1 −1 modules/recon/domains-contacts/pgp_search.py
  17. +1 −1 modules/recon/domains-credentials/pwnedlist/account_creds.py
  18. +1 −1 modules/recon/domains-credentials/pwnedlist/domain_creds.py
  19. +1 −1 modules/recon/domains-credentials/pwnedlist/domain_ispwned.py
  20. +1 −1 modules/recon/domains-hosts/baidu_site.py
  21. +1 −1 modules/recon/domains-hosts/bing_domain_api.py
  22. +1 −1 modules/recon/domains-hosts/bing_domain_web.py
  23. +13 −7 modules/recon/domains-hosts/brute_hosts.py
  24. +1 −1 modules/recon/domains-hosts/google_site_api.py
  25. +1 −1 modules/recon/domains-hosts/netcraft.py
  26. +1 −1 modules/recon/domains-hosts/shodan_hostname.py
  27. +1 −1 modules/recon/domains-hosts/ssl_san.py
  28. +1 −1 modules/recon/domains-hosts/vpnhunter.py
  29. +1 −1 modules/recon/domains-hosts/yahoo_domain.py
  30. +1 −1 modules/recon/domains-vulnerabilities/punkspider.py
  31. +1 −1 modules/recon/domains-vulnerabilities/xssed.py
  32. +1 −1 modules/recon/domains-vulnerabilities/xssposed.py
  33. +2 −0 modules/recon/hosts-hosts/ipinfodb.py
  34. +80 −0 modules/recon/hosts-hosts/ssltools.py
  35. +17 −0 modules/recon/hosts-locations/migrate_hosts.py
  36. +29 −0 modules/recon/hosts-ports/shodan_ip.py
  37. +53 −0 modules/recon/locations-pushpins/youtube.py
  38. +1 −1 modules/recon/netblocks-hosts/shodan_net.py
  39. +41 −0 modules/recon/netblocks-ports/censysio.py
  40. +6 −2 recon/core/base.py
  41. +3 −0 recon/core/framework.py
  42. +1 −1 recon/mixins/search.py
  43. +16 −12 recon/utils/parsers.py
@@ -631,8 +631,8 @@ to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.

<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
{one line to give the program's name and a brief idea of what it does.}
Copyright (C) {year} {name of author}

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@@ -652,7 +652,7 @@ Also add information on how to contact you by electronic and paper mail.
If the program does terminal interaction, make it output a short
notice like this when it starts in an interactive mode:

<program> Copyright (C) <year> <name of author>
{project} Copyright (C) {year} {fullname}
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
This is free software, and you are welcome to redistribute it
under certain conditions; type `show c' for details.
@@ -1,4 +1,4 @@
__version__ = '4.7.1'
__version__ = '4.7.2'

# ex. x.y.z
# x - Incremented for changes requiring migration. (major revision)

Large diffs are not rendered by default.

Large diffs are not rendered by default.

@@ -16,7 +16,7 @@ class Module(BaseModule):
'\tinurl:elmah.axd ext:axd intitle:"Error log for"',
'\tinurl:server-status "Apache Status"',
),
'query': 'SELECT DISTINCT host FROM hosts WHERE host IS NOT NULL ORDER BY host',
'query': 'SELECT DISTINCT host FROM hosts WHERE host IS NOT NULL',
'options': (
('download', True, True, 'download discovered files'),
('protocol', 'http', True, 'request protocol'),
@@ -9,7 +9,7 @@ class Module(BaseModule):
'name': 'Jigsaw Contact Enumerator',
'author': 'Tim Tomes (@LaNMaSteR53)',
'description': 'Harvests contacts from the Jigsaw.com API. Updates the \'contacts\' table with the results.',
'query': 'SELECT DISTINCT company FROM companies WHERE company IS NOT NULL ORDER BY company',
'query': 'SELECT DISTINCT company FROM companies WHERE company IS NOT NULL',
'options': (
('keywords', None, False, 'additional keywords to identify company'),
),
@@ -11,7 +11,7 @@ class Module(BaseModule):
'name': 'Jigsaw Authenticated Contact Enumerator',
'author': 'Travis Lee (@eelsivart)',
'description': 'Harvests contacts from Data.com using an authenticated user account. Updates the \'contacts\' table with the results. Use \'keys\' to set your jigsaw username and password before use.',
'query': 'SELECT DISTINCT company FROM companies WHERE company IS NOT NULL ORDER BY company',
'query': 'SELECT DISTINCT company FROM companies WHERE company IS NOT NULL',
}

cookiejar = CookieJar()
@@ -7,7 +7,7 @@ class Module(BaseModule):
'name': 'LinkedIn Authenticated Contact Enumerator',
'author': 'Tim Tomes (@LaNMaSteR53)',
'description': 'Harvests contacts from the LinkedIn.com API using an authenticated connections network. Updates the \'contacts\' table with the results.',
'query': 'SELECT DISTINCT company FROM companies WHERE company IS NOT NULL ORDER BY company',
'query': 'SELECT DISTINCT company FROM companies WHERE company IS NOT NULL',
}

def get_linkedin_access_token(self):
@@ -32,7 +32,7 @@ class Module(BaseModule):
'query': 'SELECT rowid, first_name, middle_name, last_name, email FROM contacts WHERE email IS NOT NULL',
'options': (
('pattern', '<fn>.<ln>', True, 'pattern applied to email'),
('overwrite', True, True, 'if set to true will update existing contact entry, otherwise it will create a new entry'),
('overwrite', False, True, 'if set to true will update existing contact entry, otherwise it will create a new entry'),
),
}

@@ -7,7 +7,7 @@ class Module(BaseModule):
'name': 'Have I been pwned? Breach Search',
'author': 'Tim Tomes (@LaNMaSteR53) & Tyler Halfpop (@tylerhalfpop)',
'description': 'Leverages the haveibeenpwned.com API to determine if email addresses are associated with breached credentials. Adds compromised email addresses to the \'credentials\' table.',
'query': 'SELECT DISTINCT email FROM contacts WHERE email IS NOT NULL ORDER BY email',
'query': 'SELECT DISTINCT email FROM contacts WHERE email IS NOT NULL',
}

def module_run(self, accounts):
@@ -10,7 +10,7 @@ class Module(BaseModule):
'comments': (
'Paste sites supported: Pastebin, Pastie, or Slexy',
),
'query': 'SELECT DISTINCT email FROM contacts WHERE email IS NOT NULL ORDER BY email',
'query': 'SELECT DISTINCT email FROM contacts WHERE email IS NOT NULL',
'options': (
('download', True, True, 'download pastes'),
),
@@ -8,7 +8,7 @@ class Module(BaseModule):
'name': 'PwnedList Validator',
'author': 'Tim Tomes (@LaNMaSteR53)',
'description': 'Leverages PwnedList.com to determine if email addresses are associated with leaked credentials. Adds compromised email addresses to the \'credentials\' table.',
'query': 'SELECT DISTINCT email FROM contacts WHERE email IS NOT NULL ORDER BY email',
'query': 'SELECT DISTINCT email FROM contacts WHERE email IS NOT NULL',
}

def module_run(self, accounts):
@@ -1,4 +1,5 @@
from recon.core.module import BaseModule
import time

class Module(BaseModule):

@@ -32,7 +33,10 @@ def module_run(self, emails):
if 'organizations' in resp.json:
for occupation in resp.json['organizations']:
if occupation['current']:
title = '%s at %s' % (occupation['title'], occupation['name'])
if 'title' in occupation:
title = '%s at %s' % (occupation['title'], occupation['name'])
else:
title = 'Employee at %s' % occupation['name']
self.output(title)
# parse demographics for region
region = None
@@ -60,3 +64,5 @@ def module_run(self, emails):
self.output('%s - Queued for search.' % email)
else:
self.output('%s - %s' % (email, resp.json['message']))
# 60 requests per minute api rate limit
time.sleep(1)
@@ -1,5 +1,4 @@
from recon.core.module import BaseModule
from cookielib import CookieJar
import StringIO
import time
import xml.etree.ElementTree
@@ -16,57 +15,22 @@ class Module(BaseModule):
'query': 'SELECT DISTINCT hash FROM credentials WHERE hash IS NOT NULL AND password IS NULL AND type IS NOT \'Adobe\'',
}

cookiejar = CookieJar()

def login(self, username, password):
url = 'https://hashes.org/login.php'
payload = { 'username': username, 'password': password, 'action':'action' }
resp = self.request(url, method='POST', payload=payload, cookiejar=self.cookiejar, redirect=False)
if resp.headers['location'] == 'index.php':
return True
return False

def module_run(self, hashes):
if not self.login(self.get_key('hashes_username'), self.get_key('hashes_password')):
self.error('Error authenticating to hashes.org.')
return
api_key = self.get_key('hashes_api')
url = 'https://hashes.org/api.php'
hash_groups = map(None, *(iter(hashes),) * 20)
for group in hash_groups:
payload = {'act':'REQUEST', 'key':api_key}
for hashstr in hashes:
payload['hash'] = hashstr
# 20 requests per minute
time.sleep(3)
# rate limit error has "data" tags
# rate limit error does not have a "hash" element
# rate limit error response for bulk requests consist of one request element
# build the payload
payload = {'do': 'check'}
group = [x for x in group if x is not None]
for i in range(0, len(group)):
payload['hash'+str(i+1)] = group[i]
resp = self.request(url, payload=payload, cookiejar=self.cookiejar)
tree = resp.xml
requests = tree.findall('request')
for request in requests:
# check for and report error conditions
# conduct check within loop to support bulk request errors
# None condition check required as tree elements with no children return False
if request.find('error') is not None:
error = request.find('error').text
# continue processing valid hashes
if 'invalid' in error:
self.verbose('Unsupported type for hash: %s' % (request.find('hash').text))
continue
# any other error results in termination
else:
self.error(error)
return
# analyze valid response
hashstr = request.find('hash').text
if request.find('found').text == 'true':
plaintext = request.find('plain').text
if hashstr != plaintext:
hashtype = request.find('type').text
self.alert('%s (%s) => %s' % (hashstr, hashtype, plaintext))
self.query('UPDATE credentials SET password=\'%s\', type=\'%s\' WHERE hash=\'%s\'' % (plaintext, hashtype, hashstr))
else:
self.verbose('Value not found for hash: %s' % (hashstr))
resp = self.request(url, payload=payload)
jsonobj = resp.json
if 'ERROR' in jsonobj:
self.verbose('%s => %s' % (hashstr, jsonobj['ERROR'].lower()))
elif jsonobj['REQUEST'] != 'FOUND':
self.verbose('%s => %s' % (hashstr, jsonobj['REQUEST'].lower()))
else:
plaintext = jsonobj[hashstr]['plain']
hashtype = jsonobj[hashstr]['algorithm']
self.alert('%s (%s) => %s' % (hashstr, hashtype, plaintext))
self.query('UPDATE credentials SET password=\'%s\', type=\'%s\' WHERE hash=\'%s\'' % (plaintext, hashtype, hashstr))
@@ -13,6 +13,9 @@ class Module(BaseModule, GoogleWebMixin):
'name': 'Meta Data Extractor',
'author': 'Tim Tomes (@LaNMaSteR53)',
'description': 'Searches for files associated with the provided domain(s) and extracts any contact related metadata.',
'comments': (
'Currently supports doc, docx, xls, xlsx, ppt, pptx, and pdf file types.',
),
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL',
'options': (
('extract', False, True, 'extract metadata from discovered files'),
@@ -41,17 +44,21 @@ def module_run(self, domains):
if ext in exts[key]:
# check to see if a parser exists for the file type
if hasattr(parsers, key+'_parser'):
func = getattr(parsers, key + '_parser')
resp = self.request(result)
# validate that the url resulted in a file
if resp.headers['content-type'].startswith('application'):
meta = func(resp.raw)
# display the extracted metadata
for key in meta:
if meta[key]:
self.output('%s: %s' % (key.title(), meta[key]))
else:
self.error('Resource not a valid file.')
try:
func = getattr(parsers, key + '_parser')
resp = self.request(result)
# validate that the url resulted in a file
if resp.headers['content-type'].startswith('application'):
meta = func(resp.raw)
# display the extracted metadata
for key in meta:
if meta[key]:
self.alert('%s: %s' % (key.title(), meta[key]))
else:
self.error('Resource not a valid file.')
except Exception:
self.print_exception()
else:
self.alert('No parser available for file type: %s' % ext)
break
self.alert('%d files found on \'%s\'.' % (len(results), domain))
@@ -10,7 +10,7 @@ class Module(BaseModule):
'comments': (
'Inspiration from theHarvester.py by Christan Martorella: cmarorella[at]edge-seecurity.com',
),
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL ORDER BY domain',
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL',
}

def module_run(self, domains):
@@ -10,7 +10,7 @@ class Module(BaseModule):
'comments': (
'API Query Cost: 1 query per request.',
),
'query': 'SELECT DISTINCT username FROM credentials WHERE username IS NOT NULL and password IS NULL ORDER BY username',
'query': 'SELECT DISTINCT username FROM credentials WHERE username IS NOT NULL and password IS NULL',
}

def module_run(self, accounts):
@@ -10,7 +10,7 @@ class Module(BaseModule):
'comments': (
'API Query Cost: 10,000 queries per request plus 1 query for each account returned.',
),
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL ORDER BY domain',
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL',
}

def module_run(self, domains):
@@ -9,7 +9,7 @@ class Module(BaseModule):
'comments': (
'API Query Cost: 1 query per request.',
),
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL ORDER BY domain',
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL',
}

def module_run(self, domains):
@@ -10,7 +10,7 @@ class Module(BaseModule):
'name': 'Baidu Hostname Enumerator',
'author': 'Tim Tomes (@LaNMaSteR53)',
'description': 'Harvests hosts from Baidu.com by using the \'site\' search operator. Updates the \'hosts\' table with the results.',
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL ORDER BY domain'
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL',
}

def module_run(self, domains):
@@ -8,7 +8,7 @@ class Module(BaseModule):
'name': 'Bing API Hostname Enumerator',
'author': 'Marcus Watson (@BranMacMuffin)',
'description': 'Leverages the Bing API and "domain:" advanced search operator to harvest hosts. Updates the \'hosts\' table with the results.',
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL ORDER BY domain',
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL',
'options': (
('limit', 0, True, 'limit total number of api requests (0 = unlimited)'),
),
@@ -11,7 +11,7 @@ class Module(BaseModule):
'name': 'Bing Hostname Enumerator',
'author': 'Tim Tomes (@LaNMaSteR53)',
'description': 'Harvests hosts from Bing.com by using the \'site\' search operator. Updates the \'hosts\' table with the results.',
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL ORDER BY domain'
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL',
}

def module_run(self, domains):
@@ -22,18 +22,19 @@ def module_run(self, domains):
resolver = self.get_resolver()
for domain in domains:
self.heading(domain, level=0)
wildcard = None
try:
answers = resolver.query('*.%s' % (domain))
self.output('Wildcard DNS entry found for \'%s\'. Cannot brute force hostnames.' % (domain))
continue
wildcard = answers.response.answer[0][0].address
self.output('Wildcard DNS entry found for \'%s\' at \'%s\'.' % (domain, wildcard))
except (dns.resolver.NoNameservers, dns.resolver.Timeout):
self.error('Invalid nameserver.')
continue
except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
self.verbose('No Wildcard DNS entry found.')
self.thread(words, domain, resolver)
self.thread(words, domain, resolver, wildcard)

def module_thread(self, word, domain, resolver):
def module_thread(self, word, domain, resolver, wildcard):
max_attempts = 3
attempt = 0
while attempt < max_attempts:
@@ -52,12 +53,17 @@ def module_thread(self, word, domain, resolver):
for rdata in answer:
if rdata.rdtype in (1, 5):
if rdata.rdtype == 1:
self.alert('%s => (A) %s - Host found!' % (host, host))
address = rdata.address
if address != wildcard:
self.alert('%s => (A) %s - Host found!' % (host, address))
self.add_hosts(host, address)
else:
self.verbose('%s => Wildcard response.' % (host))
if rdata.rdtype == 5:
cname = rdata.target.to_text()[:-1]
self.alert('%s => (CNAME) %s - Host found!' % (host, cname))
self.add_hosts(cname)
# add the host in case a CNAME exists without an A record
self.add_hosts(host)
# add the host in case a CNAME exists without an A record
self.add_hosts(host)
# break out of the loop
attempt = max_attempts
@@ -7,7 +7,7 @@ class Module(BaseModule):
'name': 'Google CSE Hostname Enumerator',
'author': 'Tim Tomes (@LaNMaSteR53)',
'description': 'Leverages the Google Custom Search Engine API to harvest hosts using the \'site\' search operator. Updates the \'hosts\' table with the results.',
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL ORDER BY domain',
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL',
}

def module_run(self, domains):
@@ -13,7 +13,7 @@ class Module(BaseModule):
'name': 'Netcraft Hostname Enumerator',
'author': 'thrapt (thrapt@gmail.com)',
'description': 'Harvests hosts from Netcraft.com. Updates the \'hosts\' table with the results.',
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL ORDER BY domain',
'query': 'SELECT DISTINCT domain FROM domains WHERE domain IS NOT NULL',
}

def module_run(self, domains):

0 comments on commit 3a211b7

Please sign in to comment.
You can’t perform that action at this time.