IP addresses can now determine their ASN from a given asn data file.

cfltools/cli.py

@@ -5,16 +5,13 @@
 
 from __future__ import print_function
 import click
-import cfltools.cflt_utils as cflt_utils  # pylint: disable=C0414
-import cfltools.config as cflt_config
 
 
 # Global Variables #
 from cfltools.utilities import APPDIR, VERSION, log_generator
-logger = log_generator(__name__)
-
-
 from cfltools import CLISession
+
+logger = log_generator(__name__)
 session = CLISession()
 
 
@@ -94,6 +91,7 @@ def logparse(filename, whois, incidentid, tor):  # pylint: disable=C0103
     #     GetWhois.run(incidentid)
     # if tor:
     #     CheckTor.run(incidentid)
+    pass
 
 
 
@@ -103,22 +101,23 @@ def createincident(incident_name):
     """
     Creates an incident to track logs associated with that event.
     """
-    from os import makedirs
-    from pathlib import Path
-    import configparser
-    config = configparser.ConfigParser()
-    config.read(APPDIR + '/cfltools.ini')
-    incident_dir = APPDIR + '/incidents/' + incident_name
-    if not Path(incident_dir).is_dir():
-        # If a directory corresponding to a particualr incident
-        # does not exist, create it. We will store our incident
-        # related-data (e.g., logfiles) in this directory.
-        makedirs(incident_dir)
-    if cflt_utils.checkIncidentNameExists(incident_name, config):
-        print("Incident identifier {} is not unique! Select a different incident name."
-              .format(incident_name))
-    else:
-        cflt_utils.generateIncident(incident_name, config)
+    # from os import makedirs
+    # from pathlib import Path
+    # import configparser
+    # config = configparser.ConfigParser()
+    # config.read(APPDIR + '/cfltools.ini')
+    # incident_dir = APPDIR + '/incidents/' + incident_name
+    # if not Path(incident_dir).is_dir():
+    #     # If a directory corresponding to a particualr incident
+    #     # does not exist, create it. We will store our incident
+    #     # related-data (e.g., logfiles) in this directory.
+    #     makedirs(incident_dir)
+    # if cflt_utils.checkIncidentNameExists(incident_name, config):
+    #     print("Incident identifier {} is not unique! Select a different incident name."
+    #           .format(incident_name))
+    # else:
+    #     cflt_utils.generateIncident(incident_name, config)
+    pass
 
 
 @cli.command()
@@ -127,14 +126,15 @@ def incidents(show):
     """
     List, modify, and remove incidents and incident identifiers.
     """
-    import sqlite3
-    import configparser
-    config = configparser.ConfigParser()
-    config.read(APPDIR + '/cfltools.ini')
-    conn = sqlite3.connect(config['USER']['db_loc'])
-    if show:
-        cflt_utils.listIncidents(conn)
-    conn.close()
+    # import sqlite3
+    # import configparser
+    # config = configparser.ConfigParser()
+    # config.read(APPDIR + '/cfltools.ini')
+    # conn = sqlite3.connect(config['USER']['db_loc'])
+    # if show:
+    #     cflt_utils.listIncidents(conn)
+    # conn.close()
+    pass
 
 
 @cli.command()
@@ -145,26 +145,27 @@ def database(saveasn, loadasn, fillmissingasn):
     """
     Database IO operations
     """
-    import sqlite3
-    import configparser
-    config = configparser.ConfigParser()
-    config.read(APPDIR + '/cfltools.ini')
-    db_connection = sqlite3.connect(config['USER']['db_loc'])
-    if saveasn:
-        from cfltools.logparse.getwhois import saveISPDBtoFile
-        from cfltools.cflt_utils import safeprompt
-        filename = safeprompt('Enter filename to save ASN database to: ',
-                              'csv')
-        saveISPDBtoFile(filename, db_connection)
-    if loadasn:
-        from cfltools.logparse.getwhois import loadISPDBfromFile
-        from cfltools.cflt_utils import safeprompt
-        filename = safeprompt('Enter filename to load ASN database from: ',
-                              'csv')
-        loadISPDBfromFile(filename, db_connection)
-    if fillmissingasn:
-        from cfltools.logparse.getwhois import getMissingASNfromUser
-        getMissingASNfromUser(db_connection)
+    # import sqlite3
+    # import configparser
+    # config = configparser.ConfigParser()
+    # config.read(APPDIR + '/cfltools.ini')
+    # db_connection = sqlite3.connect(config['USER']['db_loc'])
+    # if saveasn:
+    #     from cfltools.logparse.getwhois import saveISPDBtoFile
+    #     from cfltools.cflt_utils import safeprompt
+    #     filename = safeprompt('Enter filename to save ASN database to: ',
+    #                           'csv')
+    #     saveISPDBtoFile(filename, db_connection)
+    # if loadasn:
+    #     from cfltools.logparse.getwhois import loadISPDBfromFile
+    #     from cfltools.cflt_utils import safeprompt
+    #     filename = safeprompt('Enter filename to load ASN database from: ',
+    #                           'csv')
+    #     loadISPDBfromFile(filename, db_connection)
+    # if fillmissingasn:
+    #     from cfltools.logparse.getwhois import getMissingASNfromUser
+    #     getMissingASNfromUser(db_connection)
+    pass
 
 
 @cli.command()
@@ -173,14 +174,15 @@ def report(incident_id):
     """
     Command line / text reports related to incidents.
     """
-    import cfltools.reports.report
-    from cfltools.cflt_utils import checkIncidentNameExists
-    import configparser
-    config = configparser.ConfigParser()
-    config.read(APPDIR + '/cfltools.ini')
-    if checkIncidentNameExists(incident_id, config):
-        cfltools.reports.report.reportToCLI(incident_id, config)
-    else:
-        print('Incident {} does not exist. You can '
-              'show a list of incidents with the command '
-              'cfltools incidents --show'.format(incident_id))
+    # import cfltools.reports.report
+    # from cfltools.cflt_utils import checkIncidentNameExists
+    # import configparser
+    # config = configparser.ConfigParser()
+    # config.read(APPDIR + '/cfltools.ini')
+    # if checkIncidentNameExists(incident_id, config):
+    #     cfltools.reports.report.reportToCLI(incident_id, config)
+    # else:
+    #     print('Incident {} does not exist. You can '
+    #           'show a list of incidents with the command '
+    #           'cfltools incidents --show'.format(incident_id))
+    pass

cfltools/conftest.py

@@ -0,0 +1,3 @@
+import pytest
+from cfltools.logparse.test_logparse import *
+from cfltools.database.test_database import *

cfltools/database/test_database.py

@@ -5,7 +5,7 @@
 
 
 @pytest.fixture
-def testdatabase():
+def testdb(tmpdir):
     """
     Generates a dummy database to test db operations.
     Not implemented.
@@ -14,8 +14,6 @@ def testdatabase():
     testdb = tmpdir / 'cfltools.db'
     session = makesession(testdb)
     yield session
-    # TODO: complete this fixture
-    # develop some read/write unit tests
 
 
 def test_db_ISP():

cfltools/logparse/objects.py

@@ -2,8 +2,8 @@
 Objects for log parsing
 """
 
-
-from cfltools.utilities import Time, log_generator
+from pyasn import pyasn
+from cfltools.utilities import Time, Config, log_generator, APPDIR
 
 
 # Instantiate the logger.
@@ -34,6 +34,13 @@ def update_time(self, raw_timestamp):
         if newtime.posix() > self.latest.posix():
             self.latest = newtime
 
+    def asn(self, asndb):
+        """
+        Returns the ASN of the IP address.
+        Feed this method a precompiled ASN database (DAT file).
+        """
+        return asndb.lookup(self.ip)[0]
+
 
 class LogFile():
 
@@ -67,7 +74,6 @@ def __init__(self, filename):
         self.errors = []
         self.unique = {}
 
-
     def md5(self):
         """
         Get the MD5 checksum of the log file.
@@ -79,7 +85,7 @@ def md5(self):
 
 
 class CSVLogFile(LogFile):
-
+    """Logfile object for CSV files."""
     def __init__(self, filename):
         LogFile.__init__(self, filename)
         self._import()
@@ -232,6 +238,7 @@ def __init__(self, filename):
             raise NotImplementedError
         else:
             raise NotImplementedError
+        self.config = Config()
 
     def filetype(self):
         """

cfltools/logparse/test_logparse.py

@@ -8,6 +8,7 @@
 from datetime import datetime
 import pytest
 from cfltools.logparse import LogParser, IPAddress
+from cfltools.utilities.test_utilities import dummy_configfile
 
 
 LOGSIZE = 10
@@ -120,6 +121,17 @@ def logfile(tmpdir, iplogline):
     yield testfile
 
 
+@pytest.fixture
+def dummy_asndb(tmpdir):
+    """
+    Generates a dummy ASNDB for IP resolution testing.
+    """
+    from cfltools.utilities import asn_update
+    from pyasn import pyasn
+    asn_datfile = asn_update(tmpdir)
+    yield pyasn(asn_datfile)
+
+
 def test_open_file_and_checksum(logfile):
     """
     Verifies that a file can be opened and checksummed
@@ -204,8 +216,15 @@ def test_ipaddress_timestampconversion():
     assert ipaddr.latest.posix() == 4000 # corresponds to times[3]
 
 
+def test_ipaddress_getasn(dummy_asndb):
+    ipaddr = IPAddress('1.1.1.1', 'Wed Dec 31 16:25:00 2013')
+    assert ipaddr.asn(dummy_asndb) == dummy_asndb.lookup('1.1.1.1')[0]
+
 #TODO: unit test to check if earliest / latest times are good
 #TODO: unit test to check if time stamp conversion from various formats are good
 
+
+
+
 if __name__ == '__main__':
     pass

cfltools/objects.py

@@ -4,24 +4,35 @@
 
 import logging
 from cfltools.database import makesession
-from cfltools.utilities import Config, log_generator
+from cfltools.utilities import Config, log_generator, asn_update
 from cfltools.logparse import LogParser
+from datetime import date
 
 
+logger = log_generator(__name__).setLevel(logging.INFO)
+
 
 class Session():
     """
     High level objects that describes a CFLTools session.
     Performs integration across CFLTools submodules.
     """
-    def __init__(self):
+    def __init__(self, db=None):
         self.config = Config()
-        self.database = makesession(self.config.read("db_loc"))
-        self.logger = log_generator(__name__).setLevel(logging.INFO)
+        if db is None:
+            # This is set up to allow a dummy DB to be loaded
+            # for testing.
+            self.database = makesession(self.config.read("db_loc"))
+        else:
+            self.database = db
         self.logparser = None
 
     def logparse(self, filename):
         """Parse a log within a session"""
+        if self.config.read("asn_datfile") is None:
+            logger.info("Updating ASN file.")
+            self.config.write("asn_datfile", asn_update())
+            self.config.write("asn_lastupdate", date.today().strftime("%Y-%m-%d"))
         self.logparser = LogParser(filename)
 
     def __del__(self):
@@ -34,11 +45,25 @@ class CLISession(Session):
     A command line session of CFLTools.
     """
 
-    def __init__(self):
-        Session.__init__(self)
+    def __init__(self, db=None):
+        Session.__init__(self, db)
 
     def logparse(self, filename, incidentid=None):
         """Parse a log within a command line session."""
         Session.logparse(self, filename)
         if incidentid is None:
-            self.logger.warning("No incident ID was provided!")
+            logger.warning("No incident ID was provided!")
+
+
+class FlaskSession(Session):
+    """
+    A Flask session of CFLTools.
+    """
+    def __init__(self, db=None):
+        Session.__init__(self, db)
+
+    def logparse(self, filename, incidentid=None):
+        """Parse a log within a Flask GUI session."""
+        Session.logparse(self, filename)
+        if incidentid is None:
+            logger.warning("No incident ID was provided!")

cfltools/test_cli.py

@@ -0,0 +1,19 @@
+"""
+CLI interface tests
+"""
+import pytest
+import click
+from click.testing import CliRunner
+from .objects import CLISession
+
+
+def test_about():
+    from cfltools.cli import about
+    runner = CliRunner()
+    result = runner.invoke(about)
+    assert result.exit_code == 0
+
+def test_clisession(testdb, logfile):
+    session = CLISession(testdb)
+    session.logparse(logfile)
+

cfltools/utilities/__init__.py

@@ -8,4 +8,5 @@
 from .globals import APPDIR
 from .globals import UTILDIR
 
-from .functions import log_generator
+from .functions import log_generator
+from .functions import asn_update