Removed the google_site_api module due to the API no longer being sup…

…ported. Resolves issue #245.

Removed the freegeoip module due to the resource no longer being available. Resolves issue #290.

Added the ipstack module to replace the previously removed freegeoip module. Resolves issue #289.

Updated the geocoding modules to work with changes to the API. Resolves issue #292.

Added virustotal hostname extractor modules.

Updated the shebangs to specify Python 2 as recommended by PEP-0394.

Updated the csv reporting module to allow including a header row.

Fixed ^D for navigation.

Updated the full name parsing logic in the bing_linkedin_cache module to account for changes in formatting.

Minor cleanup of white space in the core framework module.

VERSION

@@ -1,4 +1,4 @@
-__version__ = '4.9.3'
+__version__ = '4.9.4'
 
 # ex. x.y.z
 # x - Incremented for changes requiring migration. (major revision)

modules/recon/companies-contacts/bing_linkedin_cache.py

@@ -70,7 +70,8 @@ def get_contact_info(self, cache):
             self.add_profiles(username=username, url=url, resource='LinkedIn', category='social')
 
     def parse_fullname(self, name):
-        fullname = name.split(" |")[0]
+        fullname = name.split(" -")[0]
+        fullname = fullname.split(" |")[0]
         fullname = fullname.split(",")[0]
         fname, mname, lname = self.parse_name(fullname)
         return fullname, fname, mname, lname

modules/recon/hosts-hosts/freegeoip.py → modules/recon/hosts-hosts/ipstack.py

@@ -1,33 +1,29 @@
-from recon.core.module import BaseModule
-import json
-
-class Module(BaseModule):
-
-    meta = {
-        'name': 'FreeGeoIP',
-        'author': 'Gerrit Helm (G) and Tim Tomes (@LaNMaSteR53)',
-        'description': 'Leverages the freegeoip.net API to geolocate a host by IP address. Updates the \'hosts\' table with the results.',
-        'comments': (
-            'Allows up to 10,000 queries per hour by default. Once this limit is reached, all requests will result in HTTP 403, forbidden, until the quota is cleared.',
-        ),
-        'query': 'SELECT DISTINCT ip_address FROM hosts WHERE ip_address IS NOT NULL',
-        'options': (
-            ('serverurl', 'http://freegeoip.net', True, 'overwrite server url (e.g. for local installations)'),
-        ),
-    }
-
-    def module_run(self, hosts):
-        for host in hosts:
-            url = '%s/json/%s' % (self.options['serverurl'], host)
-            resp = self.request(url)
-            if resp.json:
-                jsonobj = resp.json
-            else:
-                self.error('Invalid JSON response for \'%s\'.\n%s' % (host, resp.text))
-                continue
-            region = ', '.join([str(jsonobj[x]).title() for x in ['city', 'region_name'] if jsonobj[x]]) or None
-            country = jsonobj['country_name'].title()
-            latitude = str(jsonobj['latitude'])
-            longitude = str(jsonobj['longitude'])
-            self.output('%s - %s,%s - %s' % (host, latitude, longitude, ', '.join([x for x in [region, country] if x])))
-            self.query('UPDATE hosts SET region=?, country=?, latitude=?, longitude=? WHERE ip_address=?', (region, country, latitude, longitude, host))
+from recon.core.module import BaseModule
+import json
+
+class Module(BaseModule):
+
+    meta = {
+        'name': 'ipstack',
+        'author': 'Siarhei Harbachou (Tech.Insiders), Gerrit Helm (G) and Tim Tomes (@LaNMaSteR53)',
+        'description': 'Leverages the ipstack.com API to geolocate a host by IP address. Updates the \'hosts\' table with the results.',
+        'required_keys': ['ipstack_api'],
+        'query': 'SELECT DISTINCT ip_address FROM hosts WHERE ip_address IS NOT NULL',
+    }
+
+    def module_run(self, hosts):
+        for host in hosts:
+            api_key = self.keys.get('ipstack_api')
+            url = 'http://api.ipstack.com/%s?access_key=%s' % (host, api_key)
+            resp = self.request(url)
+            if resp.json:
+                jsonobj = resp.json
+            else:
+                self.error('Invalid JSON response for \'%s\'.\n%s' % (host, resp.text))
+                continue
+            region = ', '.join([str(jsonobj[x]).title() for x in ['city', 'region_name'] if jsonobj[x]]) or None
+            country = jsonobj['country_name'].title()
+            latitude = str(jsonobj['latitude'])
+            longitude = str(jsonobj['longitude'])
+            self.output('%s - %s,%s - %s' % (host, latitude, longitude, ', '.join([x for x in [region, country] if x])))
+            self.query('UPDATE hosts SET region=?, country=?, latitude=?, longitude=? WHERE ip_address=?', (region, country, latitude, longitude, host))

modules/recon/hosts-hosts/virustotal.py

@@ -0,0 +1,28 @@
+from recon.core.module import BaseModule
+from time import sleep
+
+class Module(BaseModule):
+
+    meta = {
+        'name': 'Virustotal domains extractor',
+        'author': 'USSC (thanks @jevalenciap)',
+        'description': 'Harvests domains from the Virustotal by using the report API. Updates the \'hosts\' table with the results.',
+        'required_keys': ['virustotal_api'],
+        'query': 'SELECT DISTINCT ip_address FROM hosts WHERE ip_address IS NOT NULL',
+        'options': (
+            ('interval', 15, True, 'interval in seconds between api requests'),
+        ),
+    }
+
+    def module_run(self, addresses):
+        key = self.get_key('virustotal_api')
+        url = 'https://www.virustotal.com/vtapi/v2/ip-address/report'
+        for ip in addresses:
+            self.heading(ip, level=0)
+            resp = self.request( url, payload = {'ip': ip, 'apikey': key} )
+            if resp.json and 'resolutions' in resp.json.keys():
+                for entry in resp.json['resolutions']:
+                    hostname = entry.get('hostname')
+                    if hostname:
+                        self.add_hosts(host=hostname, ip_address=ip)
+            sleep(self.options['interval'])

modules/recon/locations-locations/geocode.py

@@ -6,13 +6,15 @@ class Module(BaseModule):
         'name': 'Address Geocoder',
         'author': 'Quentin Kaiser (contact@quentinkaiser.be)',
         'description': 'Queries the Google Maps API to obtain coordinates for an address. Updates the \'locations\' table with the results.',
+        'required_keys': ['google_api'],
         'query': 'SELECT DISTINCT street_address FROM locations WHERE street_address IS NOT NULL',
     }
 
     def module_run(self, addresses):
+        api_key = self.keys.get('google_api')
         for address in addresses:
             self.verbose("Geocoding '%s'..." % (address))
-            payload = {'address' : address, 'sensor' : 'false'}
+            payload = {'address' : address, 'key' : api_key}
             url = 'https://maps.googleapis.com/maps/api/geocode/json'
             resp = self.request(url, payload=payload)
             # kill the module if nothing is returned

modules/recon/locations-locations/reverse_geocode.py

@@ -6,13 +6,15 @@ class Module(BaseModule):
         'name': 'Reverse Geocoder',
         'author': 'Quentin Kaiser (contact@quentinkaiser.be)',
         'description': 'Queries the Google Maps API to obtain an address from coordinates.',
+        'required_keys': ['google_api'],
         'query': 'SELECT DISTINCT latitude || \',\' || longitude FROM locations WHERE latitude IS NOT NULL AND longitude IS NOT NULL',
     }
 
     def module_run(self, points):
+        api_key = self.keys.get('google_api')
         for point in points:
             self.verbose("Reverse geocoding (%s)..." % (point))
-            payload = {'latlng' : point, 'sensor' : 'false'}
+            payload = {'latlng' : point, 'key' : api_key}
             url = 'https://maps.googleapis.com/maps/api/geocode/json'
             resp = self.request(url, payload=payload)
             # kill the module if nothing is returned

modules/recon/netblocks-hosts/virustotal.py

@@ -0,0 +1,29 @@
+from recon.core.module import BaseModule
+from time import sleep
+
+class Module(BaseModule):
+
+    meta = {
+        'name': 'Virustotal domains extractor',
+        'author': 'USSC (thanks @jevalenciap)',
+        'description': 'Harvests domains from the Virustotal by using the report API. Updates the \'hosts\' table with the results.',
+        'required_keys': ['virustotal_api'],
+        'query': 'SELECT DISTINCT netblock FROM netblocks WHERE netblock IS NOT NULL',
+        'options': (
+            ('interval', 15, True, 'interval in seconds between api requests'),
+        ),
+    }
+
+    def module_run(self, netblocks):
+        key = self.get_key('virustotal_api')
+        url = 'https://www.virustotal.com/vtapi/v2/ip-address/report'
+        for netblock in netblocks:
+            for ip in self.cidr_to_list(netblock):
+                self.heading(ip, level=0)
+                resp = self.request( url, payload = {'ip': ip, 'apikey': key} )
+                if resp.json and 'resolutions' in resp.json.keys():
+                    for entry in resp.json['resolutions']:
+                        hostname = entry.get('hostname')
+                        if hostname:
+                            self.add_hosts(host=hostname, ip_address=ip)
+                sleep(self.options['interval'])

modules/reporting/csv.py

@@ -11,21 +11,24 @@ class Module(BaseModule):
         'options': (
             ('table', 'hosts', True, 'source table of data to export'),
             ('filename', os.path.join(BaseModule.workspace, 'results.csv'), True, 'path and filename for output'),
+            ('headers', False, True, 'include column headers'),
         ),
     }
 
     def module_run(self):
         filename = self.options['filename']
         # codecs module not used because the csv module converts to ascii
         with open(filename, 'w') as outfile:
-            # build a list of table names
             table = self.options['table']
-            rows = self.query('SELECT * FROM "%s" ORDER BY 1' % (table))
+            csvwriter = csv.writer(outfile, quoting=csv.QUOTE_ALL)
+            if self.options['headers']:
+                columns = [c[0] for c in self.get_columns(table)]
+                csvwriter.writerow(columns)
             cnt = 0
+            rows = self.query('SELECT * FROM "%s" ORDER BY 1' % (table))
             for row in rows:
                 row = [x if x else '' for x in row]
                 if any(row):
                     cnt += 1
-                    csvwriter = csv.writer(outfile, quoting=csv.QUOTE_ALL)
                     csvwriter.writerow([s.encode("utf-8") for s in row])
         self.output('%d records added to \'%s\'.' % (cnt, filename))

recon-cli

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 import argparse
 import sys

recon-ng

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 import argparse
 import re

recon-rpc

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 __author__ = "Anthony Miller-Rhodes (@amillerrhodes)"
 

recon-web

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 
 from recon.core.web import app
 

recon/core/framework.py

@@ -35,19 +35,19 @@ class Options(dict):
     def __init__(self, *args, **kwargs):
         self.required = {}
         self.description = {}
-        
+
         super(Options, self).__init__(*args, **kwargs)
-           
+
     def __setitem__(self, name, value):
         super(Options, self).__setitem__(name, self._autoconvert(value))
-           
+
     def __delitem__(self, name):
         super(Options, self).__delitem__(name)
         if name in self.required:
             del self.required[name]
         if name in self.description:
             del self.description[name]
-        
+
     def _boolify(self, value):
         # designed to throw an exception if value is not a string representation of a boolean
         return {'true':True, 'false':False}[value.lower()]
@@ -68,7 +68,7 @@ def _autoconvert(self, value):
         if type(value) is int and '.' in str(orig):
             return float(orig)
         return value
-        
+
     def init_option(self, name, value=None, required=False, description=''):
         self[name] = value
         self.required[name] = required
@@ -149,7 +149,8 @@ def onecmd(self, line):
             sys.stdin = sys.__stdin__
             Framework._script = 0
             Framework._load = 0
-            return 0
+            print('')
+            return 1
         if cmd is None:
             return self.default(line)
         self.lastcmd = line