Pylint fixes

docs/conf.py

@@ -12,7 +12,7 @@
 class Mock(MagicMock):
     @classmethod
     def __getattr__(cls, name):
-            return Mock()
+        return Mock()
 
 MOCK_MODULES = ['pycurl', 'lxml', 'psycopg2']
 sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)

pyspider/database/__init__.py

@@ -5,7 +5,9 @@
 #         http://binux.me
 # Created on 2014-10-08 15:04:08
 
-import os, requests, json
+import os
+import json
+import requests
 from six.moves.urllib.parse import urlparse, parse_qs
 
 
@@ -55,48 +57,38 @@ def _connect_database(url):  # NOQA
     scheme = parsed.scheme.split('+')
     if len(scheme) == 1:
         raise Exception('wrong scheme format: %s' % parsed.scheme)
-    else:
-        engine, dbtype = scheme[0], scheme[-1]
-        other_scheme = "+".join(scheme[1:-1])
+    engine, dbtype = scheme[0], scheme[-1]
+    other_scheme = "+".join(scheme[1:-1])
 
     if dbtype not in ('taskdb', 'projectdb', 'resultdb'):
         raise LookupError('unknown database type: %s, '
                           'type should be one of ["taskdb", "projectdb", "resultdb"]', dbtype)
 
     if engine == 'mysql':
         return _connect_mysql(parsed,dbtype)
-
-    elif engine == 'sqlite':
+    if engine == 'sqlite':
         return _connect_sqlite(parsed,dbtype)
-    elif engine == 'mongodb':
+    if engine == 'mongodb':
         return _connect_mongodb(parsed,dbtype,url)
-
-    elif engine == 'sqlalchemy':
+    if engine == 'sqlalchemy':
         return _connect_sqlalchemy(parsed, dbtype, url, other_scheme)
-
-
-    elif engine == 'redis':
+    if engine == 'redis':
         if dbtype == 'taskdb':
             from .redis.taskdb import TaskDB
             return TaskDB(parsed.hostname, parsed.port,
                           int(parsed.path.strip('/') or 0))
-        else:
-            raise LookupError('not supported dbtype: %s', dbtype)
-    elif engine == 'local':
+        raise LookupError('not supported dbtype: %s', dbtype)
+    if engine == 'local':
         scripts = url.split('//', 1)[1].split(',')
         if dbtype == 'projectdb':
             from .local.projectdb import ProjectDB
             return ProjectDB(scripts)
-        else:
-            raise LookupError('not supported dbtype: %s', dbtype)
-    elif engine == 'elasticsearch' or engine == 'es':
+        raise LookupError('not supported dbtype: %s', dbtype)
+    if engine == 'elasticsearch' or engine == 'es':
         return _connect_elasticsearch(parsed, dbtype)
-
-    elif engine == 'couchdb':
+    if engine == 'couchdb':
         return _connect_couchdb(parsed, dbtype, url)
-
-    else:
-        raise Exception('unknown engine: %s' % engine)
+    raise Exception('unknown engine: %s' % engine)
 
 
 def _connect_mysql(parsed,dbtype):
@@ -115,14 +107,13 @@ def _connect_mysql(parsed,dbtype):
     if dbtype == 'taskdb':
         from .mysql.taskdb import TaskDB
         return TaskDB(**parames)
-    elif dbtype == 'projectdb':
+    if dbtype == 'projectdb':
         from .mysql.projectdb import ProjectDB
         return ProjectDB(**parames)
-    elif dbtype == 'resultdb':
+    if dbtype == 'resultdb':
         from .mysql.resultdb import ResultDB
         return ResultDB(**parames)
-    else:
-        raise LookupError
+    raise LookupError
 
 
 def _connect_sqlite(parsed,dbtype):
@@ -138,14 +129,13 @@ def _connect_sqlite(parsed,dbtype):
     if dbtype == 'taskdb':
         from .sqlite.taskdb import TaskDB
         return TaskDB(path)
-    elif dbtype == 'projectdb':
+    if dbtype == 'projectdb':
         from .sqlite.projectdb import ProjectDB
         return ProjectDB(path)
-    elif dbtype == 'resultdb':
+    if dbtype == 'resultdb':
         from .sqlite.resultdb import ResultDB
         return ResultDB(path)
-    else:
-        raise LookupError
+    raise LookupError
 
 
 def _connect_mongodb(parsed,dbtype,url):
@@ -157,14 +147,13 @@ def _connect_mongodb(parsed,dbtype,url):
     if dbtype == 'taskdb':
         from .mongodb.taskdb import TaskDB
         return TaskDB(url, **parames)
-    elif dbtype == 'projectdb':
+    if dbtype == 'projectdb':
         from .mongodb.projectdb import ProjectDB
         return ProjectDB(url, **parames)
-    elif dbtype == 'resultdb':
+    if dbtype == 'resultdb':
         from .mongodb.resultdb import ResultDB
         return ResultDB(url, **parames)
-    else:
-        raise LookupError
+    raise LookupError
 
 
 def _connect_sqlalchemy(parsed, dbtype,url, other_scheme):
@@ -174,14 +163,13 @@ def _connect_sqlalchemy(parsed, dbtype,url, other_scheme):
     if dbtype == 'taskdb':
         from .sqlalchemy.taskdb import TaskDB
         return TaskDB(url)
-    elif dbtype == 'projectdb':
+    if dbtype == 'projectdb':
         from .sqlalchemy.projectdb import ProjectDB
         return ProjectDB(url)
-    elif dbtype == 'resultdb':
+    if dbtype == 'resultdb':
         from .sqlalchemy.resultdb import ResultDB
         return ResultDB(url)
-    else:
-        raise LookupError
+    raise LookupError
 
 
 def _connect_elasticsearch(parsed, dbtype):
@@ -198,10 +186,10 @@ def _connect_elasticsearch(parsed, dbtype):
     if dbtype == 'projectdb':
         from .elasticsearch.projectdb import ProjectDB
         return ProjectDB([parsed.netloc], index=index)
-    elif dbtype == 'resultdb':
+    if dbtype == 'resultdb':
         from .elasticsearch.resultdb import ResultDB
         return ResultDB([parsed.netloc], index=index)
-    elif dbtype == 'taskdb':
+    if dbtype == 'taskdb':
         from .elasticsearch.taskdb import TaskDB
         return TaskDB([parsed.netloc], index=index)
 
@@ -220,11 +208,10 @@ def _connect_couchdb(parsed, dbtype, url):
     if dbtype == 'taskdb':
         from .couchdb.taskdb import TaskDB
         return TaskDB(url, **params)
-    elif dbtype == 'projectdb':
+    if dbtype == 'projectdb':
         from .couchdb.projectdb import ProjectDB
         return ProjectDB(url, **params)
-    elif dbtype == 'resultdb':
+    if dbtype == 'resultdb':
         from .couchdb.resultdb import ResultDB
         return ResultDB(url, **params)
-    else:
-        raise LookupError
+    raise LookupError

pyspider/database/base/projectdb.py

@@ -25,7 +25,7 @@
 }
 
 
-class ProjectDB(object):
+class ProjectDB():
     status_str = [
         'TODO',
         'STOP',
@@ -55,8 +55,7 @@ def check_update(self, timestamp, fields=None):
     def split_group(self, group, lower=True):
         if lower:
             return re.split("\W+", (group or '').lower())
-        else:
-            return re.split("\W+", group or '')
+        return re.split("\W+", group or '')
 
     def verify_project_name(self, name):
         if len(name) > 64:

pyspider/database/base/resultdb.py

@@ -17,7 +17,7 @@
 }
 
 
-class ResultDB(object):
+class ResultDB():
     """
     database for result
     """

pyspider/database/base/taskdb.py

@@ -56,7 +56,7 @@
 }
 
 
-class TaskDB(object):
+class TaskDB():
     ACTIVE = 1
     SUCCESS = 2
     FAILED = 3

pyspider/database/basedb.py

@@ -8,10 +8,9 @@
 from __future__ import unicode_literals, division, absolute_import
 
 import logging
-logger = logging.getLogger('database.basedb')
-
 from six import itervalues
 from pyspider.libs import utils
+logger = logging.getLogger('database.basedb')
 
 
 class BaseDB:

pyspider/database/couchdb/couchdbbase.py

@@ -1,7 +1,9 @@
-import time, requests, json
+import time
+
+import requests
 from requests.auth import HTTPBasicAuth
 
-class SplitTableMixin(object):
+class SplitTableMixin():
     UPDATE_PROJECTS_TIME = 10 * 60
 
     def __init__(self):
@@ -13,8 +15,7 @@ def __init__(self):
     def _collection_name(self, project):
         if self.collection_prefix:
             return "%s_%s" % (self.collection_prefix, project)
-        else:
-            return project
+        return project
 
 
     @property
@@ -92,4 +93,3 @@ def update_doc(self, db_name, doc_id, new_doc):
 
     def delete(self, url):
         return self.session.delete(url).json()
-

pyspider/database/couchdb/projectdb.py

@@ -1,4 +1,5 @@
-import time, requests, json
+import time
+import requests
 from requests.auth import HTTPBasicAuth
 from pyspider.database.base.projectdb import ProjectDB as BaseProjectDB
 

pyspider/database/couchdb/resultdb.py

@@ -1,4 +1,4 @@
-import time, json
+import time
 from pyspider.database.base.resultdb import ResultDB as BaseResultDB
 from .couchdbbase import SplitTableMixin
 
@@ -105,4 +105,4 @@ def drop(self, project):
         # drop the project
         collection_name = self._get_collection_name(project)
         url = self.base_url + collection_name
-        return self.delete(url)
+        return self.delete(url)

pyspider/database/couchdb/taskdb.py

@@ -1,4 +1,4 @@
-import json, time
+import time
 from pyspider.database.base.taskdb import TaskDB as BaseTaskDB
 from .couchdbbase import SplitTableMixin
 
@@ -107,4 +107,4 @@ def drop_database(self):
     def drop(self, project):
         collection_name = self._get_collection_name(project)
         url = self.base_url + collection_name
-        return self.delete(url)
+        return self.delete(url)

pyspider/database/local/projectdb.py

@@ -7,9 +7,10 @@
 
 import os
 import re
-import six
 import glob
 import logging
+import six
+
 
 from pyspider.database.base.projectdb import ProjectDB as BaseProjectDB
 
@@ -44,7 +45,7 @@ def load_scripts(self):
 
     def _build_project(self, filename):
         try:
-            with open(filename) as fp:
+            with open(filename, encoding='utf-8') as fp:
                 script = fp.read()
             m = self.rate_re.search(script)
             if m:

pyspider/database/mongodb/mongodbbase.py

@@ -8,14 +8,13 @@
 import time
 
 
-class SplitTableMixin(object):
+class SplitTableMixin():
     UPDATE_PROJECTS_TIME = 10 * 60
 
     def _collection_name(self, project):
         if self.collection_prefix:
             return "%s.%s" % (self.collection_prefix, project)
-        else:
-            return project
+        return project
 
     @property
     def projects(self):

pyspider/database/mysql/mysqlbase.py

@@ -9,7 +9,7 @@
 import mysql.connector
 
 
-class MySQLMixin(object):
+class MySQLMixin():
     maxlimit = 18446744073709551615
 
     @property
@@ -26,14 +26,13 @@ def dbcur(self):
             return self.conn.cursor()
 
 
-class SplitTableMixin(object):
+class SplitTableMixin():
     UPDATE_PROJECTS_TIME = 10 * 60
 
     def _tablename(self, project):
         if self.__tablename__:
             return '%s_%s' % (self.__tablename__, project)
-        else:
-            return project
+        return project
 
     @property
     def projects(self):

pyspider/database/mysql/resultdb.py

@@ -6,9 +6,9 @@
 # Created on 2014-10-13 22:02:57
 
 import re
-import six
 import time
 import json
+import six
 import mysql.connector
 
 from pyspider.libs import utils

pyspider/database/mysql/taskdb.py

@@ -7,9 +7,10 @@
 
 
 import re
-import six
 import time
 import json
+import six
+
 import mysql.connector
 
 from pyspider.libs import utils
@@ -97,7 +98,7 @@ def get_task(self, project, taskid, fields=None):
         return None
 
     def status_count(self, project):
-        result = dict()
+        result = {}
         if project not in self.projects:
             self._list_project()
         if project not in self.projects:

pyspider/database/redis/__init__.py

@@ -4,4 +4,3 @@
 # Author: Binux<roy@binux.me>
 #         http://binux.me
 # Created on 2015-05-17 01:34:21
-