Merge pull request #748 from p-l-/pylint-fixes

Pylint fixes

.travis.yml

@@ -188,7 +188,7 @@ before_script:
   - export BRO_SAMPLES=`pwd`/usr/local/bro/testing
   - ivre --version; echo; bro --version; echo; nmap --version
 
-script: (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" = 2.6 || test "$TRAVIS_PYTHON_VERSION" = 3.3 || (flake8 --ignore=F401,E402 doc/conf.py && flake8 --ignore=W504 setup.py bin/ivre && flake8 --ignore=E402,W504 tests/tests.py && flake8 --ignore=W504 ivre/ && echo "flake8 OK (except W504)")) && (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.7 || (codespell  --ignore-words=.travis/codespell_ignore `git ls-files | grep -vE '^web/static/(doc|an|bs|d3|jq|lk)/|^data/|\.(png|gif|svg)$'` && echo "codespell OK")) && (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.7 || (pylint -e all -d abstract-method,access-member-before-definition,arguments-differ,attribute-defined-outside-init,bad-continuation,broad-except,cyclic-import,deprecated-method,deprecated-module,dict-values-not-iterating,duplicate-code,eq-without-hash,exec-used,fixme,function-redefined,global-statement,global-variable-undefined,import-error,invalid-name,locally-disabled,missing-docstring,next-method-called,no-absolute-import,no-member,non-parent-init-called,nonzero-method,no-self-use,old-division,protected-access,redefined-argument-from-local,redefined-outer-name,reload-builtin,round-builtin,singleton-comparison,subprocess-popen-preexec-fn,super-init-not-called,suppressed-message,too-few-public-methods,too-many-ancestors,too-many-arguments,too-many-boolean-expressions,too-many-branches,too-many-instance-attributes,too-many-lines,too-many-locals,too-many-nested-blocks,too-many-public-methods,too-many-return-statements,too-many-statements,unexpected-keyword-arg,unsubscriptable-object,unsupported-membership-test,unused-argument,useless-object-inheritance,useless-super-delegation,wrong-import-order,wrong-import-position ivre && echo "pylint OK")) && cd tests/ && coverage erase && coverage run --parallel-mode tests.py --coverage && coverage combine && coverage report
+script: (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" = 2.6 || test "$TRAVIS_PYTHON_VERSION" = 3.3 || (flake8 --ignore=F401,E402 doc/conf.py && flake8 --ignore=W504 setup.py bin/ivre && flake8 --ignore=E402,W504 tests/tests.py && flake8 --ignore=W504 ivre/ && echo "flake8 OK (except W504)")) && (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.7 || (codespell  --ignore-words=.travis/codespell_ignore `git ls-files | grep -vE '^web/static/(doc|an|bs|d3|jq|lk)/|^data/|\.(png|gif|svg)$'` && echo "codespell OK")) && (test "$DB" != "maxmind" || test "$TRAVIS_PYTHON_VERSION" != 3.7 || (pylint -e all -d abstract-method,arguments-differ,attribute-defined-outside-init,bad-continuation,broad-except,cyclic-import,deprecated-method,deprecated-module,duplicate-code,exec-used,fixme,function-redefined,global-statement,global-variable-undefined,import-error,invalid-name,locally-disabled,missing-docstring,no-absolute-import,no-member,non-parent-init-called,no-self-use,old-division,protected-access,redefined-outer-name,reload-builtin,round-builtin,singleton-comparison,subprocess-popen-preexec-fn,super-init-not-called,suppressed-message,too-few-public-methods,too-many-ancestors,too-many-arguments,too-many-boolean-expressions,too-many-branches,too-many-instance-attributes,too-many-lines,too-many-locals,too-many-nested-blocks,too-many-public-methods,too-many-return-statements,too-many-statements,unsubscriptable-object,unsupported-membership-test,unused-argument,useless-object-inheritance,wrong-import-order,wrong-import-position ivre && echo "pylint OK")) && cd tests/ && coverage erase && coverage run --parallel-mode tests.py --coverage && coverage combine && coverage report
 
 after_success:
   - codecov

ivre/db/__init__.py

@@ -1438,9 +1438,8 @@ def store_scan_json(self, fname, filehash=None,
         with utils.open_file(fname) as fdesc:
             for line in fdesc:
                 host = self.json2dbrec(json.loads(line.decode()))
-                for fname in ["_id"]:
-                    if fname in host:
-                        del host[fname]
+                if "_id" in host:
+                    del host["_id"]
                 host["scanid"] = filehash
                 if categories:
                     host["categories"] = categories
@@ -2518,6 +2517,7 @@ def __init__(self, url=None, urls=None):
     @property
     def nmap(self):
         try:
+            # pylint: disable=access-member-before-definition
             return self._nmap
         except AttributeError:
             pass
@@ -2527,6 +2527,7 @@ def nmap(self):
     @property
     def passive(self):
         try:
+            # pylint: disable=access-member-before-definition
             return self._passive
         except AttributeError:
             pass
@@ -2536,6 +2537,7 @@ def passive(self):
     @property
     def data(self):
         try:
+            # pylint: disable=access-member-before-definition
             return self._data
         except AttributeError:
             pass
@@ -2545,6 +2547,7 @@ def data(self):
     @property
     def agent(self):
         try:
+            # pylint: disable=access-member-before-definition
             return self._agent
         except AttributeError:
             pass
@@ -2554,6 +2557,7 @@ def agent(self):
     @property
     def flow(self):
         try:
+            # pylint: disable=access-member-before-definition
             return self._flow
         except AttributeError:
             pass
@@ -2563,6 +2567,7 @@ def flow(self):
     @property
     def view(self):
         try:
+            # pylint: disable=access-member-before-definition
             return self._view
         except AttributeError:
             pass

ivre/db/mongo.py

@@ -46,7 +46,7 @@
 
 import bson
 from future.builtins import bytes, range, zip
-from future.utils import viewitems, with_metaclass
+from future.utils import viewitems, viewvalues, with_metaclass
 from past.builtins import basestring
 from pymongo.errors import BulkWriteError
 import pymongo
@@ -1458,12 +1458,12 @@ def flt_cond(p):
                             p.get('port') == port and
                             p.get('protocol') == protocol)
         updated = False
-        for port in host.get('ports', []):
-            if not flt_cond(port):
+        for portdoc in host.get('ports', []):
+            if not flt_cond(portdoc):
                 continue
-            screenwords = utils.screenwords(self.getscreenshot(port))
+            screenwords = utils.screenwords(self.getscreenshot(portdoc))
             if screenwords is not None:
-                port['screenwords'] = screenwords
+                portdoc['screenwords'] = screenwords
                 updated = True
         if updated:
             self.db[self.columns[self.column_hosts]].update(
@@ -4840,7 +4840,7 @@ def count(self, flt):
         destinations = 0
         flows = self.db[self.columns[self.column_flow]].count(flt)
         if flows > 0:
-            sources = self.db[self.columns[self.column_flow]].aggregate([
+            sources = next(self.db[self.columns[self.column_flow]].aggregate([
                 {'$match': flt},
                 {
                     '$group': {
@@ -4857,23 +4857,25 @@ def count(self, flt):
                     '_id': None,
                     'count': {'$sum': 1}
                 }}
-            ]).next()['count']
-
-            destinations = self.db[self.columns[self.column_flow]].aggregate([
-                {'$match': flt},
-                {
-                    '$group': {
-                        '_id': {
-                            'dst_addr_0': '$dst_addr_0',
-                            'dst_addr_1': '$dst_addr_1'
-                        },
-                    }
-                },
-                {'$group': {
-                    '_id': None,
-                    'count': {'$sum': 1}
-                }}
-            ]).next()['count']
+            ]))['count']
+
+            destinations = next(
+                self.db[self.columns[self.column_flow]].aggregate([
+                    {'$match': flt},
+                    {
+                        '$group': {
+                            '_id': {
+                                'dst_addr_0': '$dst_addr_0',
+                                'dst_addr_1': '$dst_addr_1'
+                            },
+                        }
+                    },
+                    {'$group': {
+                        '_id': None,
+                        'count': {'$sum': 1}
+                    }}
+                ])
+            )['count']
 
         return {'clients': sources, 'servers': destinations, 'flows': flows}
 
@@ -5231,9 +5233,9 @@ def cursor2json_graph(cls, cursor, mode, timeline):
                         host["data"]["lastseen"])
                 else:
                     hosts[host["id"]] = host
-        g["nodes"] = hosts.values()
+        g["nodes"] = list(viewvalues(hosts))
         if mode in ["flow_map", "talk_map"]:
-            g["edges"] = edges.values()
+            g["edges"] = list(viewvalues(edges))
         return g
 
     @classmethod

ivre/db/sql/__init__.py

@@ -30,7 +30,7 @@
 import re
 
 
-from builtins import int, range
+from builtins import int, object, range
 from future.utils import PY3, viewitems, viewvalues
 from past.builtins import basestring
 from sqlalchemy import and_, cast, column, create_engine, delete, desc, func, \
@@ -453,9 +453,6 @@ class SQLDBFlow(SQLDB, DBFlow):
     table_layout = namedtuple("flow_layout", ['flow'])
     tables = table_layout(Flow)
 
-    def __init__(self, url):
-        super(SQLDBFlow, self).__init__(url)
-
     @staticmethod
     def query(*args, **kargs):
         raise NotImplementedError()
@@ -526,12 +523,13 @@ def fltor(flt1, flt2):
 class ActiveFilter(Filter):
 
     def __init__(self, main=None, hostname=None, category=None, port=None,
-                 script=None, trace=None):
+                 script=None, tables=None, trace=None):
         self.main = main
         self.hostname = [] if hostname is None else hostname
         self.category = [] if category is None else category
         self.port = [] if port is None else port
         self.script = [] if script is None else script
+        self.tables = tables  # default value is handled in the subclasses
         self.trace = [] if trace is None else trace
 
     @property
@@ -670,20 +668,30 @@ class NmapFilter(ActiveFilter):
 
     def __init__(self, main=None, hostname=None, category=None, port=None,
                  script=None, tables=None, trace=None):
-        super(NmapFilter, self).__init__(main=main, hostname=hostname,
-                                         category=category, port=port,
-                                         script=script, trace=trace)
-        self.tables = SQLDBNmap.tables if tables is None else tables
+        super(NmapFilter, self).__init__(
+            main=main,
+            hostname=hostname,
+            category=category,
+            port=port,
+            script=script,
+            tables=SQLDBNmap.tables if tables is None else tables,
+            trace=trace,
+        )
 
 
 class ViewFilter(ActiveFilter):
 
     def __init__(self, main=None, hostname=None, category=None, port=None,
                  script=None, tables=None, trace=None):
-        super(ViewFilter, self).__init__(main=main, hostname=hostname,
-                                         category=category, port=port,
-                                         script=script, trace=trace)
-        self.tables = SQLDBView.tables if tables is None else tables
+        super(ViewFilter, self).__init__(
+            main=main,
+            hostname=hostname,
+            category=category,
+            port=port,
+            script=script,
+            tables=SQLDBView.tables if tables is None else tables,
+            trace=trace,
+        )
 
 
 class SQLDBActive(SQLDB, DBActive):
@@ -1681,9 +1689,6 @@ class SQLDBView(SQLDBActive, DBView):
 
     base_filter = ViewFilter
 
-    def __init__(self, url):
-        super(SQLDBView, self).__init__(url)
-
     def store_or_merge_host(self, host):
         # FIXME: may cause performance issues
         self.start_store_hosts()
@@ -1713,7 +1718,7 @@ def all_queries(self):
             "tables": self.tables,
         }
 
-    def __nonzero__(self):
+    def __bool__(self):
         return self.main is not None
 
     def copy(self):
@@ -1793,9 +1798,6 @@ class SQLDBPassive(SQLDB, DBPassive):
 
     base_filter = PassiveFilter
 
-    def __init__(self, url):
-        super(SQLDBPassive, self).__init__(url)
-
     def count(self, flt):
         return self.db.execute(
             flt.query(

ivre/db/sql/postgres.py

@@ -40,9 +40,6 @@
 
 class PostgresDB(SQLDB):
 
-    def __init__(self, url):
-        super(PostgresDB, self).__init__(url)
-
     @staticmethod
     def ip2internal(addr):
         return utils.force_int2ip(addr)
@@ -119,8 +116,8 @@ def append(self, query):
     def commit(self, query=None, renew=True):
         if query is None:
             last = len(self.queries) - 1
-            for i, query in enumerate(list(self.queries)):
-                self.commit(query=query, renew=True if i < last else renew)
+            for i, q_query in enumerate(list(self.queries)):
+                self.commit(query=q_query, renew=True if i < last else renew)
             return
         q_query, params = self.queries.pop(query)
         self.conn.execute(q_query, *params)
@@ -145,16 +142,11 @@ def close(self):
 
 
 class PostgresDBFlow(PostgresDB, SQLDBFlow):
-
-    def __init__(self, url):
-        super(PostgresDBFlow, self).__init__(url)
+    pass
 
 
 class PostgresDBActive(PostgresDB, SQLDBActive):
 
-    def __init__(self, url):
-        super(PostgresDBActive, self).__init__(url)
-
     def _migrate_schema_10_11(self):
         """Converts a record from version 10 to version 11.
 
@@ -1005,9 +997,6 @@ def _features_port_get(self, features, flt, yieldall, use_service,
 
 class PostgresDBNmap(PostgresDBActive, SQLDBNmap):
 
-    def __init__(self, url):
-        super(PostgresDBNmap, self).__init__(url)
-
     def store_scan_doc(self, scan):
         scan = scan.copy()
         if 'start' in scan:
@@ -1151,9 +1140,6 @@ def store_hosts(self, hosts):
 
 class PostgresDBView(PostgresDBActive, SQLDBView):
 
-    def __init__(self, url):
-        super(PostgresDBView, self).__init__(url)
-
     def _store_host(self, host):
         addr = self.ip2internal(host['addr'])
         info = host.get('infos')

ivre/scanengine.py

@@ -169,7 +169,7 @@ def __init__(self, targets, category, agents, outputpath,
                  visiblecategory=None, maxfeed=None, sleep=2,
                  storedown=True):
         self.targets = targets
-        self.targiter = targets.__iter__()
+        self.targiter = iter(targets)
         self.category = category
         for agent in agents:
             agent.campaigns.append(self)

ivre/tools/passivereconworker.py

@@ -135,9 +135,9 @@ def worker(progname, directory, sensor=None):
         else:
             utils.LOGGER.debug('  ... KO')
     # SHUTDOWN
-    for sensor in procs:
-        procs[sensor].stdin.close()
-        procs[sensor].wait()
+    for sensorjob in procs:
+        procs[sensorjob].stdin.close()
+        procs[sensorjob].wait()
 
 
 def main():

ivre/tools/runscans.py

@@ -214,7 +214,7 @@ def call_nmap(options, xmlprocess, targets,
     fcntl.fcntl(procout, fcntl.F_SETFL, procoutfl | os.O_NONBLOCK)
     toread = [proc.stdout]
     towrite = [proc.stdin]
-    targiter = targets.__iter__()
+    targiter = iter(targets)
     while toread:
         # print("ENTERING SELECT")
         rlist, wlist = select.select(toread, towrite, [])[:2]
@@ -467,7 +467,7 @@ def main():
                 pass
         exit(0)
     elif args.output == 'ListAllRand':
-        targiter = targets.__iter__()
+        targiter = iter(targets)
         try:
             for target in targiter:
                 print(ivre.utils.int2ip(target))