Merge f98eaab into 2835bb9

.pylintrc

@@ -69,12 +69,11 @@ disable=too-few-public-methods,
         abstract-method,
         bad-continuation,
         invalid-name,
-        wrong-import-order,
         too-few-public-methods,
         keyword-arg-before-vararg,
         arguments-differ,
-	attribute-defined-outside-init,
-	no-self-use
+        attribute-defined-outside-init,
+	relative-beyond-top-level
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option

.travis.yml

@@ -14,6 +14,7 @@ install:
     - pip install -r test/requirements.txt
     - pip install -r scripts/importer/requirements.txt
     - pip install coverage coveralls
+    - pip install pylint
 script:
     - test/travis_script.sh
 addons:

backend/application.py

@@ -21,7 +21,6 @@
 import settings
 from modules.browser import utils
 
-
 def build_dataset_structure(dataset_version, user=None, dataset=None):
     if dataset is None:
         dataset = dataset_version.dataset
@@ -45,7 +44,7 @@ def build_dataset_structure(dataset_version, user=None, dataset=None):
 
 
 class QuitHandler(handlers.UnsafeHandler):
-    def get(self):
+    def get(self):  # pylint: disable=no-self-use
         ioloop = tornado.ioloop.IOLoop.instance()
         ioloop.stop()
 
@@ -66,7 +65,7 @@ def get(self):
         version = None
         beacon = None
         try:
-            url = self.get_argument('url')
+            url = self.get_argument('url')  # pylint: disable=no-value-for-parameter
             match = re.match(".*/dataset/([^/]+)(/version/([^/]+))?", url)
             if match:
                 dataset = match.group(1)
@@ -256,7 +255,7 @@ def post(self, dataset, ds_version=None):
             logging.error(f"Could not clean old linkhashes: {err}")
 
         self.finish({'hash': link_hash.hash,
-                     'expires_on': link_hash.expires_on.strftime("%Y-%m-%d %H:%M")})
+                     'expires_on': link_hash.expires_on.strftime("%Y-%m-%d %H:%M")})  # pylint: disable=no-member
 
 
 class DatasetFiles(handlers.AuthorizedHandler):
@@ -300,7 +299,7 @@ def get(self, dataset, ds_version=None):
 
         ret = {
             'collections': collections,
-            'study':       db.build_dict_from_row(dataset.study)
+            'study': db.build_dict_from_row(dataset.study)
         }
         ret['study']['publication_date'] = ret['study']['publication_date'].strftime('%Y-%m-%d')
 
@@ -389,9 +388,9 @@ def post(self, dataset):
         user = self.current_user
         dataset = db.get_dataset(dataset)
 
-        affiliation = self.get_argument("affiliation", strip=False)
-        country = self.get_argument("country", strip=False)
-        newsletter = self.get_argument("newsletter", strip=False)
+        affiliation = self.get_argument("affiliation", strip=False)  # pylint: disable=no-value-for-parameter
+        country = self.get_argument("country", strip=False)  # pylint: disable=no-value-for-parameter
+        newsletter = self.get_argument("newsletter", strip=False)  # pylint: disable=no-value-for-parameter
 
         user.affiliation = affiliation
         user.country = country
@@ -453,16 +452,16 @@ def post(self, dataset, email):
         try:
             msg = MIMEMultipart()
             msg['to'] = email
-            msg['from'] = settings.from_address
+            msg['from'] = settings.from_address  # pylint: disable=no-member
             msg['subject'] = 'Swefreq access granted to {}'.format(dataset.short_name)
-            msg.add_header('reply-to', settings.reply_to_address)
+            msg.add_header('reply-to', settings.reply_to_address)  # pylint: disable=no-member
             body = """You now have access to the {} dataset
 
     Please visit https://swefreq.nbis.se/dataset/{}/download to download files.
             """.format(dataset.full_name, dataset.short_name)
             msg.attach(MIMEText(body, 'plain'))
 
-            server = smtplib.SMTP(settings.mail_server)
+            server = smtplib.SMTP(settings.mail_server)  # pylint: disable=no-member
             server.sendmail(msg['from'], [msg['to']], msg.as_string())
         except smtplib.SMTPException as err:
             logging.error(f"Email error: {err}")
@@ -473,7 +472,7 @@ def post(self, dataset, email):
 
 
 class RevokeUser(handlers.AdminHandler):
-    def post(self, dataset, email):
+    def post(self, dataset, email):  # pylint: disable=no-self-use
         dataset, _ = utils.parse_dataset(dataset)
         with db.database.atomic():
             dataset = db.get_dataset(dataset)
@@ -626,7 +625,7 @@ def post(self):
              .execute())
         except db.SFTPUser.DoesNotExist:
             # if there is no user, insert the user in the database
-            (db.SFTPUser.insert(user=self.current_user,
+            (db.SFTPUser.insert(user=self.current_user,    # pylint: disable=no-value-for-parameter
                                 user_uid=db.get_next_free_uid(),
                                 user_name=username,
                                 password_hash=passwd_hash,
@@ -636,7 +635,7 @@ def post(self):
                      'expires': expires.strftime("%Y-%m-%d %H:%M"),
                      'password': password})
 
-    def generate_password(self, size: int = 12) -> str:
+    def generate_password(self, size: int = 12) -> str:  # pylint: disable=no-self-use
         """
         Generates a password of length 'size', comprised of random lowercase and
         uppercase letters, and numbers.

backend/auth.py

@@ -10,17 +10,16 @@
 from handlers import BaseHandler
 
 
-
 class DeveloperLoginHandler(BaseHandler):
     def get(self):
         if not self.get_argument("user", False):
             self.send_error(status_code=403)
         elif not self.get_argument("email", False):
             self.send_error(status_code=403)
 
-        self.set_secure_cookie('user', self.get_argument("user"))
-        self.set_secure_cookie('email', self.get_argument("email"))
-        self.set_secure_cookie('identity', self.get_argument("email"))
+        self.set_secure_cookie('user', self.get_argument("user"))  # pylint: disable=no-value-for-parameter
+        self.set_secure_cookie('email', self.get_argument("email"))  # pylint: disable=no-value-for-parameter
+        self.set_secure_cookie('identity', self.get_argument("email"))  # pylint: disable=no-value-for-parameter
         self.finish()
 
 
@@ -55,7 +54,7 @@ async def get(self):
                 self.redirect("/security_warning")
                 return
 
-            user_token = await self.get_user_token(self.get_argument('code'))
+            user_token = await self.get_user_token(self.get_argument('code'))  # pylint: disable=no-value-for-parameter
             user = await self.get_user(user_token["access_token"])
 
             try:
@@ -76,11 +75,11 @@ async def get(self):
             self.redirect(redirect)
 
         elif self.get_argument("error", False):
-            logging.error("Elixir error: {}".format(self.get_argument("error")))
-            logging.error(" Description: {}".format(self.get_argument("error_description")))
+            logging.error("Elixir error: {}".format(self.get_argument("error")))  # pylint: disable=no-value-for-parameter
+            logging.error(" Description: {}".format(self.get_argument("error_description")))  # pylint: disable=no-value-for-parameter
 
-            self.set_user_msg(f"Elixir Error: ,{self.get_argument('error')} " +
-                              f"{self.get_argument('error_description')}")
+            self.set_user_msg(f"Elixir Error: ,{self.get_argument('error')} " +   # pylint: disable=no-value-for-parameter
+                              f"{self.get_argument('error_description')}")  # pylint: disable=no-value-for-parameter
             self.redirect("/error")
 
         else:
@@ -115,8 +114,7 @@ async def get_user_token(self, code):
         client_id = self.settings['elixir_oauth']['id']
         secret = self.settings['elixir_oauth']['secret']
 
-        authorization = base64.b64encode(bytes(f"{client_id}:{secret}",
-                                               'ascii')).decode('ascii')
+        authorization = base64.b64encode(bytes(f"{client_id}: {secret}", 'ascii')).decode('ascii')
 
         response = await http.fetch(self._OAUTH_ACCESS_TOKEN_URL,
                                     method="POST",

backend/db.py

@@ -17,13 +17,14 @@
 
 import settings
 
+# pylint: disable=no-member
 database = PostgresqlExtDatabase(settings.psql_name,
                                  user=settings.psql_user,
                                  password=settings.psql_pass,
                                  host=settings.psql_host,
                                  port=settings.psql_port,
                                  register_hstore=False)
-
+# pylint: enable=no-member
 
 class BaseModel(Model):
     class Meta:

backend/handlers.py

@@ -21,7 +21,7 @@ class directly but from either SafeHandler or UnsafeHandler
     """
     def prepare(self):
         # Make sure we have the xsrf_token, this will generate the xsrf cookie if it isn't set
-        self.xsrf_token
+        self.xsrf_token  # pylint: disable=pointless-statement
         if db.database.is_closed():
             try:
                 db.database.connect()

backend/modules/browser/browser_handlers.py

@@ -34,7 +34,7 @@ def get(self, dataset: str, query: str, ds_version: str = None):
 class Download(handlers.UnsafeHandler):
     """Download variants in CSV format."""
 
-    def get(self, dataset: str, datatype: str, item: str,
+    def get(self, dataset: str, datatype: str, item: str,  # pylint: disable=too-many-arguments
             ds_version: str = None, filter_type: str = None):
         """
         Download variants in CSV format.
@@ -283,6 +283,7 @@ def get(self, dataset: str, variant: str, ds_version: str = None):
             dataset (str): short name of the dataset
             variant (str): variant in the format chrom-pos-ref-alt
         """
+        # pylint: disable=too-many-locals,too-many-branches,too-many-statements
         dataset, ds_version = utils.parse_dataset(dataset, ds_version)
 
         ret = {'variant': {}}
@@ -343,7 +344,7 @@ def get(self, dataset: str, variant: str, ds_version: str = None):
                       'sift': annotation['SIFT'].rstrip("()0123456789"),
                       'polyphen': annotation['PolyPhen'].rstrip("()0123456789"),
                       'canonical': annotation['CANONICAL'],
-                      'modification': annotation['HGVSp'].split(":")[1] if ':' in annotation['HGVSp'] else None}]
+                      'modification': annotation['HGVSp'].split(":")[1] if ':' in annotation['HGVSp'] else None}]  # pylint: disable=line-too-long
 
         # Dataset frequencies.
         # This is reported per variable in the database data, with dataset

backend/modules/browser/lookups.py

@@ -69,6 +69,7 @@ def get_awesomebar_result(dataset: str, query: str, ds_version: str = None) -> t
         tuple: (datatype, identifier)
 
     """
+    # pylint: disable=too-many-return-statements,too-many-branches
     query = query.strip()
 
     # Parse Variant types
@@ -84,35 +85,32 @@ def get_awesomebar_result(dataset: str, query: str, ds_version: str = None) -> t
     # Gene
     try:
         gene = get_gene_by_name(dataset, query)
+        return 'gene', gene['gene_id']
     except error.NotFoundError:
         pass
-    else:
-        return 'gene', gene['gene_id']
+
     # Capital letters for all other queries
     query = query.upper()
     try:
         gene = get_gene_by_name(dataset, query)
+        return 'gene', gene['gene_id']
     except error.NotFoundError:
         pass
-    else:
-        return 'gene', gene['gene_id']
 
     # Ensembl formatted queries
     if query.startswith('ENS'):
         # Gene
         try:
             gene = get_gene(dataset, query)
+            return 'gene', gene['gene_id']
         except error.NotFoundError:
             pass
-        else:
-            return 'gene', gene['gene_id']
         # Transcript
         try:
             transcript = get_transcript(dataset, query)
+            return 'transcript', transcript['transcript_id']
         except error.NotFoundError:
             pass
-        else:
-            return 'transcript', transcript['transcript_id']
 
     # Region and variant queries
     query = query[3:] if query.startswith('CHR') else query
@@ -348,7 +346,7 @@ def get_genes_in_region(dataset: str, chrom: str, start_pos: int,
     return genes
 
 
-def get_raw_variant(dataset: str, pos: int, chrom: str, ref: str,
+def get_raw_variant(dataset: str, pos: int, chrom: str, ref: str,  # pylint: disable=too-many-arguments
                     alt: str, ds_version: str = None) -> dict:
     """
     Retrieve variant by position and change.
@@ -480,7 +478,7 @@ def get_transcripts_in_gene_by_dbid(gene_dbid: int) -> list:
                                           .dicts())]
 
 
-def get_variant(dataset: str, pos: int, chrom: str, ref: str,
+def get_variant(dataset: str, pos: int, chrom: str, ref: str,  # pylint: disable=too-many-arguments
                 alt: str, ds_version: str = None) -> dict:
     """
     Retrieve variant by position and change.

backend/modules/browser/route.py

@@ -1,14 +1,15 @@
 from . import browser_handlers as handlers
 
+# pylint: disable=line-too-long
+
 # Browser links
-routes = [(r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/gene/(?P<gene>[^/]+)" ,                              handlers.GetGene),
-          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/region/(?P<region>[^\/]+)",                          handlers.GetRegion),
-          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/transcript/(?P<transcript>[^/]+)",                   handlers.GetTranscript),
-          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/variant/(?P<variant>[^/]+)",                         handlers.GetVariant),
-          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/variants/(?P<datatype>[^/]+)/(?P<item>[^/]+)",       handlers.GetVariants),
-          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/coverage/(?P<datatype>[^/]+)/(?P<item>[^/]+)",       handlers.GetCoverage),
-          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/coverage_pos/(?P<datatype>[^/]+)/(?P<item>[^/]+)",   handlers.GetCoveragePos),
-          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/autocomplete/(?P<query>[^/]+)",                      handlers.Autocomplete),
-          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/search/(?P<query>[^/]+)",                            handlers.Search),
-          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/download/(?P<datatype>[^/]+)/(?P<item>[^/]+)(?:/filter/(?P<filter_type>[^/]+))?",       handlers.Download),
-        ]
+routes = [(r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/gene/(?P<gene>[^/]+)", handlers.GetGene),
+          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/region/(?P<region>[^\/]+)", handlers.GetRegion),
+          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/transcript/(?P<transcript>[^/]+)", handlers.GetTranscript),
+          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/variant/(?P<variant>[^/]+)", handlers.GetVariant),
+          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/variants/(?P<datatype>[^/]+)/(?P<item>[^/]+)", handlers.GetVariants),
+          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/coverage/(?P<datatype>[^/]+)/(?P<item>[^/]+)", handlers.GetCoverage),
+          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/coverage_pos/(?P<datatype>[^/]+)/(?P<item>[^/]+)", handlers.GetCoveragePos),
+          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/autocomplete/(?P<query>[^/]+)", handlers.Autocomplete),
+          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/search/(?P<query>[^/]+)", handlers.Search),
+          (r"/api/dataset/(?P<dataset>[^/]+)/(?:version/(?P<ds_version>[^/]+)/)?browser/download/(?P<datatype>[^/]+)/(?P<item>[^/]+)(?:/filter/(?P<filter_type>[^/]+))?", handlers.Download)]

backend/modules/browser/tests/test_browser_handlers.py

@@ -1,10 +1,13 @@
 """
 Test the browser handlers
 """
-import requests
 import json
 
-BASE_URL="http://localhost:4000"
+import requests
+
+BASE_URL = "http://localhost:4000"
+
+# pylint: disable=line-too-long
 
 def test_get_autocomplete():
     """
@@ -28,7 +31,7 @@ def test_download():
     data_type = 'transcript'
     data_item = 'ENST00000438441'
     response = requests.get('{}/api/dataset/{}/browser/download/{}/{}'.format(BASE_URL, dataset, data_type, data_item))
-    assert len(response.text.split('\n')) == 180 # header + 178 + \n
+    assert len(response.text.split('\n')) == 180  # header + 178 + \n
     response = requests.get('{}/api/dataset/{}/browser/download/{}/{}/filter/all~false'.format(BASE_URL, dataset, data_type, data_item))
     import logging
     logging.error(response.text.split('\n'))

backend/modules/browser/utils.py

@@ -359,11 +359,7 @@ def get_variant_list(dataset: str, datatype: str, item: str, ds_version: str = N
 
     elif datatype == 'transcript':
         variants = lookups.get_variants_in_transcript(dataset, item, ds_version)
-
-    if datatype == 'transcript':
         transcript = lookups.get_transcript(dataset, item, ds_version)
-        if not transcript:
-            return {}
         refgene = transcript['gene_id']
 
     if variants:

backend/route.py

@@ -17,6 +17,7 @@
 define("develop", default=False, help="Run in develop environment", type=bool)
 
 # Setup the Tornado Application
+# pylint: disable=no-member
 tornado_settings = {"debug": False,
                     "cookie_secret": swefreq_settings.cookie_secret,
                     "login_url": "/login",
@@ -27,7 +28,9 @@
                     },
                     "xsrf_cookies": True,
                     "template_path": "templates/"}
+# pylint: enable=no-member
 
+# pylint: disable=line-too-long
 
 class Application(tornado.web.Application):
     def __init__(self, settings):

backend/settings.py

@@ -24,7 +24,7 @@
 
 elixir = json_settings["elixir"]
 
-## Generated with base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes)
+# Generated with base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes)
 cookie_secret = json_settings["cookieSecret"]
 
 # PostgreSQL settings