Attempt 1 to fix some travis failures

cfltools/logparse/objects.py

@@ -2,12 +2,15 @@
 Objects for log parsing
 """
 
+import pdb
 from pyasn import pyasn
-from cfltools.utilities import Time, Config, log_generator, APPDIR
+from datetime import date
+from cfltools.utilities import Time, Config, log_generator, asn_update
 
 
 # Instantiate the logger.
 logger = log_generator(__name__)
+config = Config()
 
 
 class IPAddress():
@@ -21,6 +24,7 @@ def __init__(self, ip, raw_timestamp):
         self.earliest = Time(raw_timestamp)
         self.latest = self.earliest
         self.occurances = 1
+        self.asn = None
 
     def update_time(self, raw_timestamp):
         """
@@ -34,12 +38,18 @@ def update_time(self, raw_timestamp):
         if newtime.posix() > self.latest.posix():
             self.latest = newtime
 
-    def asn(self, asndb):
+    def get_asn(self, asndb):
         """
         Returns the ASN of the IP address.
         Feed this method a precompiled ASN database (DAT file).
         """
-        return asndb.lookup(self.ip)[0]
+        if self.asn is None:
+            self.asn = asndb.lookup(self.ip)[0]
+        return self.asn
+
+    def values(self):
+        """Returns a dict of the values in this IP object."""
+        return {'IP':self.ip, 'ASN':self.asn}
 
 
 class LogFile():
@@ -66,6 +76,17 @@ def _parse(self):
                 self.unique[ip].update_time(timestamp)
             else:
                 self.unique[ip] = IPAddress(ip, timestamp)
+        asndb = self.get_asndb()
+        errors = []
+        for entry in self.unique.values():
+            try:
+                entry.asn = entry.get_asn(asndb)
+            except ValueError:
+                logger.warning("Could not find ASN for IP [%s] due to ValueError. " + \
+                               "This may be because it is the header row of a logfile.", entry.ip)
+                errors.append(entry.ip)
+        for badip in errors:
+            del self.unique[badip]
 
     def __init__(self, filename):
         logger.debug("Importing %s.", filename)
@@ -83,6 +104,15 @@ def md5(self):
             data = file.read()
         return md5(data.encode('utf-8')).hexdigest()
 
+    def get_asndb(self):
+        """Get an ASN database"""
+        if config.read("asn_datfile") is None:
+            logger.warning("No ASN data file detected! Creating one...")
+            config.write("asn_datfile", asn_update())
+            config.write("asn_lastupdate", date.today().strftime("%Y-%m-%d"))
+        asndb = pyasn(config.read("asn_datfile"))
+        return asndb
+
 
 class CSVLogFile(LogFile):
     """Logfile object for CSV files."""
@@ -238,7 +268,6 @@ def __init__(self, filename):
             raise NotImplementedError
         else:
             raise NotImplementedError
-        self.config = Config()
 
     def filetype(self):
         """
@@ -252,3 +281,7 @@ def filetype(self):
             return 'csv'
         logger.error("File %s is not supported.", self.filename)
         return 'unsupported'
+
+    def unique(self):
+        """Return an array of unique IPs in a file"""
+        return self.logfile.unique

cfltools/logparse/test_logparse.py

@@ -217,8 +217,9 @@ def test_ipaddress_timestampconversion():
 
 
 def test_ipaddress_getasn(dummy_asndb):
+    """Test IPAddress() asn lookup method"""
     ipaddr = IPAddress('1.1.1.1', 'Wed Dec 31 16:25:00 2013')
-    assert ipaddr.asn(dummy_asndb) == dummy_asndb.lookup('1.1.1.1')[0]
+    assert ipaddr.get_asn(dummy_asndb) == dummy_asndb.lookup('1.1.1.1')[0]
 
 #TODO: unit test to check if earliest / latest times are good
 #TODO: unit test to check if time stamp conversion from various formats are good

cfltools/utilities/objects.py

@@ -47,18 +47,20 @@ class Config():
     def __init__(self, configfile_loc=APPDIR/'cfltools.ini'):
         self.parser = ConfigParser()
         self.configfile = configfile_loc
+        self.parser.read(self.configfile)
         default_appfolder = APPDIR
         default_database = APPDIR / 'cfltools.db'
-        self.parser['DEFAULT'] = {'appfolder': default_appfolder.as_posix(),
-                                  'db_loc': default_database.as_posix(),
-                                  'max_tor_requests': '100',
-                                  'max_whois_requests': '100'
-                                  }
-        with open(self.configfile, 'w') as file:
+        if 'DEFAULT' not in self.parser:
+            logger.debug("Writing defaults to configfile %s", self.configfile)
+            self.parser['DEFAULT'] = {'appfolder': default_appfolder.as_posix(),
+                                      'db_loc': default_database.as_posix(),
+                                      'max_tor_requests': '100',
+                                      'max_whois_requests': '100'
+                                      }
             self.parser.write(file)
         if 'USER' not in self.parser:
             self.parser['USER'] = {}
-            with open(self.configfile, 'w') as file:
+            with open(self.configfile, 'a') as file:
                 self.parser.write(file)
 
     def read(self, attr):
@@ -86,6 +88,6 @@ def write(self, attr, newvalue):
         """
         # parser.read(self.configfile)
         self.parser['USER'][attr] = newvalue
-        with open(self.configfile, 'w') as file:
+        with open(self.configfile, 'a') as file:
             self.parser.write(file)
         logger.info("Changed %s to %s", attr, newvalue)

requirements

@@ -5,3 +5,4 @@ netaddr
 appdirs
 dateparser
 requests
+sqlalchemy