From 2753fb1330b4e422e63fcd99d3ab7ca42a4c5286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 06:42:13 +0200 Subject: [PATCH 01/25] Bad newline. --- backend/auth.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/auth.py b/backend/auth.py index b8637cfbe..e01911ee3 100644 --- a/backend/auth.py +++ b/backend/auth.py @@ -115,8 +115,7 @@ async def get_user_token(self, code): client_id = self.settings['elixir_oauth']['id'] secret = self.settings['elixir_oauth']['secret'] - authorization = base64.b64encode(bytes(f"{client_id}:{secret}", - 'ascii')).decode('ascii') + authorization = base64.b64encode(bytes(f"{client_id}: {secret}", 'ascii')).decode('ascii') response = await http.fetch(self._OAUTH_ACCESS_TOKEN_URL, method="POST", From df64521bbe08fd5a1686eb7e9d291c6fe7aa3e73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 06:58:45 +0200 Subject: [PATCH 02/25] Minor whitespace fix. --- backend/modules/browser/tests/test_browser_handlers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/modules/browser/tests/test_browser_handlers.py b/backend/modules/browser/tests/test_browser_handlers.py index b69797afb..9d251d3e3 100644 --- a/backend/modules/browser/tests/test_browser_handlers.py +++ b/backend/modules/browser/tests/test_browser_handlers.py @@ -4,7 +4,7 @@ import requests import json -BASE_URL="http://localhost:4000" +BASE_URL = "http://localhost:4000" def test_get_autocomplete(): """ From 195722836db91ead84a1b3fc7ea4c37f8ef5850f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 07:24:01 +0200 Subject: [PATCH 03/25] Comment fix. --- backend/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/settings.py b/backend/settings.py index c2e255ca0..d9482b8b2 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -24,7 +24,7 @@ elixir = json_settings["elixir"] -## Generated with base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes) +# Generated with base64.b64encode(uuid.uuid4().bytes + uuid.uuid4().bytes) cookie_secret = json_settings["cookieSecret"] # PostgreSQL settings From 5932f62c59e8e106422fc14816f0724b16422c4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 07:38:35 +0200 Subject: [PATCH 04/25] Newline fixes, comment space. --- backend/application.py | 1 - backend/auth.py | 1 - backend/modules/browser/tests/test_browser_handlers.py | 3 ++- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/application.py b/backend/application.py index bf566c5bd..110db1ba2 100644 --- a/backend/application.py +++ b/backend/application.py @@ -21,7 +21,6 @@ import settings from modules.browser import utils - def build_dataset_structure(dataset_version, user=None, dataset=None): if dataset is None: dataset = dataset_version.dataset diff --git a/backend/auth.py b/backend/auth.py index e01911ee3..64aa5f622 100644 --- a/backend/auth.py +++ b/backend/auth.py @@ -10,7 +10,6 @@ from handlers import BaseHandler - class DeveloperLoginHandler(BaseHandler): def get(self): if not self.get_argument("user", False): diff --git a/backend/modules/browser/tests/test_browser_handlers.py b/backend/modules/browser/tests/test_browser_handlers.py index 9d251d3e3..5f9b485f9 100644 --- a/backend/modules/browser/tests/test_browser_handlers.py +++ b/backend/modules/browser/tests/test_browser_handlers.py @@ -6,6 +6,7 @@ BASE_URL = "http://localhost:4000" + def test_get_autocomplete(): """ Test GetAutocomplete.get() @@ -28,7 +29,7 @@ def test_download(): data_type = 'transcript' data_item = 'ENST00000438441' response = requests.get('{}/api/dataset/{}/browser/download/{}/{}'.format(BASE_URL, dataset, data_type, data_item)) - assert len(response.text.split('\n')) == 180 # header + 178 + \n + assert len(response.text.split('\n')) == 180 # header + 178 + \n response = requests.get('{}/api/dataset/{}/browser/download/{}/{}/filter/all~false'.format(BASE_URL, dataset, data_type, data_item)) import logging logging.error(response.text.split('\n')) From 8d60f0f8c67928cf3848b31705b50ed5bace4bf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 08:13:35 +0200 Subject: [PATCH 05/25] Add row-specific disables of pylint tests. --- backend/application.py | 26 +++++++++---------- backend/auth.py | 16 ++++++------ backend/db.py | 3 ++- backend/handlers.py | 2 +- backend/modules/browser/browser_handlers.py | 2 +- backend/modules/browser/route.py | 2 ++ .../browser/tests/test_browser_handlers.py | 1 + backend/route.py | 3 +++ 8 files changed, 31 insertions(+), 24 deletions(-) diff --git a/backend/application.py b/backend/application.py index 110db1ba2..99bd49672 100644 --- a/backend/application.py +++ b/backend/application.py @@ -44,7 +44,7 @@ def build_dataset_structure(dataset_version, user=None, dataset=None): class QuitHandler(handlers.UnsafeHandler): - def get(self): + def get(self): # pylint: disable=no-self-use ioloop = tornado.ioloop.IOLoop.instance() ioloop.stop() @@ -65,7 +65,7 @@ def get(self): version = None beacon = None try: - url = self.get_argument('url') + url = self.get_argument('url') # pylint: disable=no-value-for-parameter match = re.match(".*/dataset/([^/]+)(/version/([^/]+))?", url) if match: dataset = match.group(1) @@ -255,7 +255,7 @@ def post(self, dataset, ds_version=None): logging.error(f"Could not clean old linkhashes: {err}") self.finish({'hash': link_hash.hash, - 'expires_on': link_hash.expires_on.strftime("%Y-%m-%d %H:%M")}) + 'expires_on': link_hash.expires_on.strftime("%Y-%m-%d %H:%M")}) # pylint: disable=no-member class DatasetFiles(handlers.AuthorizedHandler): @@ -299,7 +299,7 @@ def get(self, dataset, ds_version=None): ret = { 'collections': collections, - 'study': db.build_dict_from_row(dataset.study) + 'study': db.build_dict_from_row(dataset.study) } ret['study']['publication_date'] = ret['study']['publication_date'].strftime('%Y-%m-%d') @@ -388,9 +388,9 @@ def post(self, dataset): user = self.current_user dataset = db.get_dataset(dataset) - affiliation = self.get_argument("affiliation", strip=False) - country = self.get_argument("country", strip=False) - newsletter = self.get_argument("newsletter", strip=False) + affiliation = self.get_argument("affiliation", strip=False) # pylint: disable=no-value-for-parameter + country = self.get_argument("country", strip=False) # pylint: disable=no-value-for-parameter + newsletter = self.get_argument("newsletter", strip=False) # pylint: disable=no-value-for-parameter user.affiliation = affiliation user.country = country @@ -452,16 +452,16 @@ def post(self, dataset, email): try: msg = MIMEMultipart() msg['to'] = email - msg['from'] = settings.from_address + msg['from'] = settings.from_address # pylint: disable=no-member msg['subject'] = 'Swefreq access granted to {}'.format(dataset.short_name) - msg.add_header('reply-to', settings.reply_to_address) + msg.add_header('reply-to', settings.reply_to_address) # pylint: disable=no-member body = """You now have access to the {} dataset Please visit https://swefreq.nbis.se/dataset/{}/download to download files. """.format(dataset.full_name, dataset.short_name) msg.attach(MIMEText(body, 'plain')) - server = smtplib.SMTP(settings.mail_server) + server = smtplib.SMTP(settings.mail_server) # pylint: disable=no-member server.sendmail(msg['from'], [msg['to']], msg.as_string()) except smtplib.SMTPException as err: logging.error(f"Email error: {err}") @@ -472,7 +472,7 @@ def post(self, dataset, email): class RevokeUser(handlers.AdminHandler): - def post(self, dataset, email): + def post(self, dataset, email): # pylint: disable=no-self-use dataset, _ = utils.parse_dataset(dataset) with db.database.atomic(): dataset = db.get_dataset(dataset) @@ -625,7 +625,7 @@ def post(self): .execute()) except db.SFTPUser.DoesNotExist: # if there is no user, insert the user in the database - (db.SFTPUser.insert(user=self.current_user, + (db.SFTPUser.insert(user=self.current_user, # pylint: disable=no-value-for-parameter user_uid=db.get_next_free_uid(), user_name=username, password_hash=passwd_hash, @@ -635,7 +635,7 @@ def post(self): 'expires': expires.strftime("%Y-%m-%d %H:%M"), 'password': password}) - def generate_password(self, size: int = 12) -> str: + def generate_password(self, size: int = 12) -> str: # pylint: disable=no-self-use """ Generates a password of length 'size', comprised of random lowercase and uppercase letters, and numbers. diff --git a/backend/auth.py b/backend/auth.py index 64aa5f622..eaefa63fa 100644 --- a/backend/auth.py +++ b/backend/auth.py @@ -17,9 +17,9 @@ def get(self): elif not self.get_argument("email", False): self.send_error(status_code=403) - self.set_secure_cookie('user', self.get_argument("user")) - self.set_secure_cookie('email', self.get_argument("email")) - self.set_secure_cookie('identity', self.get_argument("email")) + self.set_secure_cookie('user', self.get_argument("user")) # pylint: disable=no-value-for-parameter + self.set_secure_cookie('email', self.get_argument("email")) # pylint: disable=no-value-for-parameter + self.set_secure_cookie('identity', self.get_argument("email")) # pylint: disable=no-value-for-parameter self.finish() @@ -54,7 +54,7 @@ async def get(self): self.redirect("/security_warning") return - user_token = await self.get_user_token(self.get_argument('code')) + user_token = await self.get_user_token(self.get_argument('code')) # pylint: disable=no-value-for-parameter user = await self.get_user(user_token["access_token"]) try: @@ -75,11 +75,11 @@ async def get(self): self.redirect(redirect) elif self.get_argument("error", False): - logging.error("Elixir error: {}".format(self.get_argument("error"))) - logging.error(" Description: {}".format(self.get_argument("error_description"))) + logging.error("Elixir error: {}".format(self.get_argument("error"))) # pylint: disable=no-value-for-parameter + logging.error(" Description: {}".format(self.get_argument("error_description"))) # pylint: disable=no-value-for-parameter - self.set_user_msg(f"Elixir Error: ,{self.get_argument('error')} " + - f"{self.get_argument('error_description')}") + self.set_user_msg(f"Elixir Error: ,{self.get_argument('error')} " + # pylint: disable=no-value-for-parameter + f"{self.get_argument('error_description')}") # pylint: disable=no-value-for-parameter self.redirect("/error") else: diff --git a/backend/db.py b/backend/db.py index 1c6958cc8..679bdc2cf 100644 --- a/backend/db.py +++ b/backend/db.py @@ -17,13 +17,14 @@ import settings +# pylint: disable=no-member database = PostgresqlExtDatabase(settings.psql_name, user=settings.psql_user, password=settings.psql_pass, host=settings.psql_host, port=settings.psql_port, register_hstore=False) - +# pylint: enable=no-member class BaseModel(Model): class Meta: diff --git a/backend/handlers.py b/backend/handlers.py index 21809a6fc..fcb5710b0 100644 --- a/backend/handlers.py +++ b/backend/handlers.py @@ -21,7 +21,7 @@ class directly but from either SafeHandler or UnsafeHandler """ def prepare(self): # Make sure we have the xsrf_token, this will generate the xsrf cookie if it isn't set - self.xsrf_token + self.xsrf_token # pylint: disable=pointless-statement if db.database.is_closed(): try: db.database.connect() diff --git a/backend/modules/browser/browser_handlers.py b/backend/modules/browser/browser_handlers.py index 082c7cd2f..bf7d9c412 100644 --- a/backend/modules/browser/browser_handlers.py +++ b/backend/modules/browser/browser_handlers.py @@ -343,7 +343,7 @@ def get(self, dataset: str, variant: str, ds_version: str = None): 'sift': annotation['SIFT'].rstrip("()0123456789"), 'polyphen': annotation['PolyPhen'].rstrip("()0123456789"), 'canonical': annotation['CANONICAL'], - 'modification': annotation['HGVSp'].split(":")[1] if ':' in annotation['HGVSp'] else None}] + 'modification': annotation['HGVSp'].split(":")[1] if ':' in annotation['HGVSp'] else None}] # pylint: disable=line-too-long # Dataset frequencies. # This is reported per variable in the database data, with dataset diff --git a/backend/modules/browser/route.py b/backend/modules/browser/route.py index 222309535..d49f2dc62 100755 --- a/backend/modules/browser/route.py +++ b/backend/modules/browser/route.py @@ -1,5 +1,7 @@ from . import browser_handlers as handlers +# pylint: disable=line-too-long + # Browser links routes = [(r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/gene/(?P[^/]+)" , handlers.GetGene), (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/region/(?P[^\/]+)", handlers.GetRegion), diff --git a/backend/modules/browser/tests/test_browser_handlers.py b/backend/modules/browser/tests/test_browser_handlers.py index 5f9b485f9..34979f91b 100644 --- a/backend/modules/browser/tests/test_browser_handlers.py +++ b/backend/modules/browser/tests/test_browser_handlers.py @@ -6,6 +6,7 @@ BASE_URL = "http://localhost:4000" +# pylint: disable=line-too-long def test_get_autocomplete(): """ diff --git a/backend/route.py b/backend/route.py index 81f5f9b8b..f21e3e31d 100644 --- a/backend/route.py +++ b/backend/route.py @@ -17,6 +17,7 @@ define("develop", default=False, help="Run in develop environment", type=bool) # Setup the Tornado Application +# pylint: disable=no-member tornado_settings = {"debug": False, "cookie_secret": swefreq_settings.cookie_secret, "login_url": "/login", @@ -27,7 +28,9 @@ }, "xsrf_cookies": True, "template_path": "templates/"} +# pylint: enable=no-member +# pylint: disable=line-too-long class Application(tornado.web.Application): def __init__(self, settings): From c5d7db296f1dc0db26cec2e7b7cb4e608e8c2f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 08:42:01 +0200 Subject: [PATCH 06/25] Swap import order. --- backend/modules/browser/tests/test_browser_handlers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/modules/browser/tests/test_browser_handlers.py b/backend/modules/browser/tests/test_browser_handlers.py index 34979f91b..e0c2c025e 100644 --- a/backend/modules/browser/tests/test_browser_handlers.py +++ b/backend/modules/browser/tests/test_browser_handlers.py @@ -1,9 +1,10 @@ """ Test the browser handlers """ -import requests import json +import requests + BASE_URL = "http://localhost:4000" # pylint: disable=line-too-long From 573a1c7b3380d7348d46bf500aab8d2a22ef131a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 08:48:13 +0200 Subject: [PATCH 07/25] Fix whitespace. --- backend/modules/browser/route.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/backend/modules/browser/route.py b/backend/modules/browser/route.py index d49f2dc62..6e011009b 100755 --- a/backend/modules/browser/route.py +++ b/backend/modules/browser/route.py @@ -3,14 +3,13 @@ # pylint: disable=line-too-long # Browser links -routes = [(r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/gene/(?P[^/]+)" , handlers.GetGene), - (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/region/(?P[^\/]+)", handlers.GetRegion), - (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/transcript/(?P[^/]+)", handlers.GetTranscript), - (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/variant/(?P[^/]+)", handlers.GetVariant), - (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/variants/(?P[^/]+)/(?P[^/]+)", handlers.GetVariants), - (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/coverage/(?P[^/]+)/(?P[^/]+)", handlers.GetCoverage), - (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/coverage_pos/(?P[^/]+)/(?P[^/]+)", handlers.GetCoveragePos), - (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/autocomplete/(?P[^/]+)", handlers.Autocomplete), - (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/search/(?P[^/]+)", handlers.Search), - (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/download/(?P[^/]+)/(?P[^/]+)(?:/filter/(?P[^/]+))?", handlers.Download), - ] +routes = [(r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/gene/(?P[^/]+)", handlers.GetGene), + (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/region/(?P[^\/]+)", handlers.GetRegion), + (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/transcript/(?P[^/]+)", handlers.GetTranscript), + (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/variant/(?P[^/]+)", handlers.GetVariant), + (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/variants/(?P[^/]+)/(?P[^/]+)", handlers.GetVariants), + (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/coverage/(?P[^/]+)/(?P[^/]+)", handlers.GetCoverage), + (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/coverage_pos/(?P[^/]+)/(?P[^/]+)", handlers.GetCoveragePos), + (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/autocomplete/(?P[^/]+)", handlers.Autocomplete), + (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/search/(?P[^/]+)", handlers.Search), + (r"/api/dataset/(?P[^/]+)/(?:version/(?P[^/]+)/)?browser/download/(?P[^/]+)/(?P[^/]+)(?:/filter/(?P[^/]+))?", handlers.Download)] From 3cbe7b418a44c08f937c7f56c8ae554d73ec92b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 08:51:40 +0200 Subject: [PATCH 08/25] Reenable wrong-import-order and no-self-use. --- .pylintrc | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.pylintrc b/.pylintrc index eb0218b05..7b493e82f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -69,12 +69,10 @@ disable=too-few-public-methods, abstract-method, bad-continuation, invalid-name, - wrong-import-order, too-few-public-methods, keyword-arg-before-vararg, arguments-differ, - attribute-defined-outside-init, - no-self-use + attribute-defined-outside-init # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option From 165aab0b9d5d1e3f1b485a599a77856ca000ac0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 09:59:18 +0200 Subject: [PATCH 09/25] Remove unneeded if. --- backend/modules/browser/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/modules/browser/utils.py b/backend/modules/browser/utils.py index fab28332d..32612474b 100644 --- a/backend/modules/browser/utils.py +++ b/backend/modules/browser/utils.py @@ -359,8 +359,6 @@ def get_variant_list(dataset: str, datatype: str, item: str, ds_version: str = N elif datatype == 'transcript': variants = lookups.get_variants_in_transcript(dataset, item, ds_version) - - if datatype == 'transcript': transcript = lookups.get_transcript(dataset, item, ds_version) if not transcript: return {} From e0180a0859f7b5d38057eb4819f482f9c880e27b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 10:18:53 +0200 Subject: [PATCH 10/25] Remove check as the situation will never occur. If get_transcripts can't find the transcript, it will raise an exception and not return anything. --- backend/modules/browser/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/modules/browser/utils.py b/backend/modules/browser/utils.py index 32612474b..67055efd5 100644 --- a/backend/modules/browser/utils.py +++ b/backend/modules/browser/utils.py @@ -360,8 +360,6 @@ def get_variant_list(dataset: str, datatype: str, item: str, ds_version: str = N elif datatype == 'transcript': variants = lookups.get_variants_in_transcript(dataset, item, ds_version) transcript = lookups.get_transcript(dataset, item, ds_version) - if not transcript: - return {} refgene = transcript['gene_id'] if variants: From 9d2e8552094f998a5facae7f5dcb9467cff03710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 10:28:04 +0200 Subject: [PATCH 11/25] Add row/block specific disables of pylint tests. --- backend/modules/browser/browser_handlers.py | 3 ++- backend/modules/browser/lookups.py | 18 ++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/backend/modules/browser/browser_handlers.py b/backend/modules/browser/browser_handlers.py index bf7d9c412..568064cd9 100644 --- a/backend/modules/browser/browser_handlers.py +++ b/backend/modules/browser/browser_handlers.py @@ -34,7 +34,7 @@ def get(self, dataset: str, query: str, ds_version: str = None): class Download(handlers.UnsafeHandler): """Download variants in CSV format.""" - def get(self, dataset: str, datatype: str, item: str, + def get(self, dataset: str, datatype: str, item: str, # pylint: disable=too-many-arguments ds_version: str = None, filter_type: str = None): """ Download variants in CSV format. @@ -283,6 +283,7 @@ def get(self, dataset: str, variant: str, ds_version: str = None): dataset (str): short name of the dataset variant (str): variant in the format chrom-pos-ref-alt """ + # pylint: disable=too-many-locals,too-many-branches,too-many-statements dataset, ds_version = utils.parse_dataset(dataset, ds_version) ret = {'variant': {}} diff --git a/backend/modules/browser/lookups.py b/backend/modules/browser/lookups.py index ac7344c8c..d538fab57 100644 --- a/backend/modules/browser/lookups.py +++ b/backend/modules/browser/lookups.py @@ -69,6 +69,7 @@ def get_awesomebar_result(dataset: str, query: str, ds_version: str = None) -> t tuple: (datatype, identifier) """ + # pylint: disable=too-many-return-statements,too-many-branches query = query.strip() # Parse Variant types @@ -84,35 +85,32 @@ def get_awesomebar_result(dataset: str, query: str, ds_version: str = None) -> t # Gene try: gene = get_gene_by_name(dataset, query) + return 'gene', gene['gene_id'] except error.NotFoundError: pass - else: - return 'gene', gene['gene_id'] + # Capital letters for all other queries query = query.upper() try: gene = get_gene_by_name(dataset, query) + return 'gene', gene['gene_id'] except error.NotFoundError: pass - else: - return 'gene', gene['gene_id'] # Ensembl formatted queries if query.startswith('ENS'): # Gene try: gene = get_gene(dataset, query) + return 'gene', gene['gene_id'] except error.NotFoundError: pass - else: - return 'gene', gene['gene_id'] # Transcript try: transcript = get_transcript(dataset, query) + return 'transcript', transcript['transcript_id'] except error.NotFoundError: pass - else: - return 'transcript', transcript['transcript_id'] # Region and variant queries query = query[3:] if query.startswith('CHR') else query @@ -348,7 +346,7 @@ def get_genes_in_region(dataset: str, chrom: str, start_pos: int, return genes -def get_raw_variant(dataset: str, pos: int, chrom: str, ref: str, +def get_raw_variant(dataset: str, pos: int, chrom: str, ref: str, # pylint: disable=too-many-arguments alt: str, ds_version: str = None) -> dict: """ Retrieve variant by position and change. @@ -480,7 +478,7 @@ def get_transcripts_in_gene_by_dbid(gene_dbid: int) -> list: .dicts())] -def get_variant(dataset: str, pos: int, chrom: str, ref: str, +def get_variant(dataset: str, pos: int, chrom: str, ref: str, # pylint: disable=too-many-arguments alt: str, ds_version: str = None) -> dict: """ Retrieve variant by position and change. From c77fd612fc81769c6a1581caa11bc35a0bef0647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 10:32:58 +0200 Subject: [PATCH 12/25] Add pylint checking to backend. --- test/travis_script.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/travis_script.sh b/test/travis_script.sh index bcb67dcd9..225b03d05 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -152,7 +152,6 @@ sort "$BASE/tests/data/reference.psql" > ref.psql # compare dump to reference diff sdump.psql ref.psql - RETURN_VALUE=$((RETURN_VALUE + $?)) echo '>>> Test 6. Reading manta file' @@ -169,6 +168,9 @@ psql -U postgres -h 127.0.0.1 -p 5433 "$DBNAME" -c "select chrom_id, pos, ref, a diff mates_res.txt "$BASE/tests/data/mates_reference.txt" RETURN_VALUE=$((RETURN_VALUE + $?)) +echo '>>> Code evaluation' +pylint backend +RETURN_VALUE=$((RETURN_VALUE + $?)) echo '>>> Finalising: Combine coverage' From d1c5e96aec715bfbad4bb1b0966ffd324262f3e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 10:39:39 +0200 Subject: [PATCH 13/25] Install pylint in travis. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 678dd835a..4595cbe93 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ install: - pip install -r test/requirements.txt - pip install -r scripts/importer/requirements.txt - pip install coverage coveralls + - pip install pylint script: - test/travis_script.sh addons: From 742de70775051f5f21b375236737539666716858 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 10:59:18 +0200 Subject: [PATCH 14/25] Pylint fixes in watch_frontend. --- scripts/watch_frontend.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/scripts/watch_frontend.py b/scripts/watch_frontend.py index 25bfff8c4..760dcaafb 100644 --- a/scripts/watch_frontend.py +++ b/scripts/watch_frontend.py @@ -1,9 +1,8 @@ -import inotify.adapters import re import subprocess import os import logging - +import inotify.adapters def files_to_watch(): r = re.compile("^/code/frontend/(templates|src)") @@ -18,7 +17,7 @@ def comb(*args): def install_node_deps(): os.chdir('frontend') - subprocess.call(['npm','install']) + subprocess.call(['npm', 'install']) os.chdir('..') def main(): @@ -31,12 +30,12 @@ def main(): grepper = comb(files_to_watch(), events_to_watch()) while True: - changes = list( filter(grepper, i.event_gen(timeout_s=0.5, yield_nones=False)) ) + changes = list(filter(grepper, i.event_gen(timeout_s=0.5, yield_nones=False))) if changes: logging.info("Files updated rerunning") for c in changes: (_, type_names, path, filename) = c - logging.info(" PATH=[{}] FILENAME=[{}] EVENT_TYPES={}".format(path, filename, type_names)) + logging.info(f" PATH=[{path}] FILENAME=[{filename}] EVENT_TYPES={type_names}") subprocess.call(['make']) From e8e0c68b7f123a406d54d5306bfe756800995310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 11:07:29 +0200 Subject: [PATCH 15/25] Pylint fixes in add_picture_to_db. --- scripts/add_picture_to_db.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/scripts/add_picture_to_db.py b/scripts/add_picture_to_db.py index 819c11732..3b6f6eb4b 100755 --- a/scripts/add_picture_to_db.py +++ b/scripts/add_picture_to_db.py @@ -30,18 +30,16 @@ def add_image(dataset_pk, image): data = f.read() try: - mimetype = infer_mimetype( image ) + mimetype = infer_mimetype(image) except NoMimetypeException: print("Can't find mime type for <{}>".format(image)) sys.exit(2) print(len(data)) with db.database.atomic(): - db.DatasetLogo.create( - dataset = dataset, - mimetype = mimetype, - data = data - ) + db.DatasetLogo.create(dataset=dataset, + mimetype=mimetype, + data=data) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Insert a picture into the database') From fae1400f40597429d677c9c3572a8a65e6cbd0cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 11:07:41 +0200 Subject: [PATCH 16/25] Pylint fixes in checkRequests. --- scripts/checkRequests.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/scripts/checkRequests.py b/scripts/checkRequests.py index 40a66ad0b..bbb570fa3 100644 --- a/scripts/checkRequests.py +++ b/scripts/checkRequests.py @@ -1,28 +1,29 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText import logging -import peewee +import secrets import smtplib +import peewee + import db -import secrets def send_email(): - msg = MIMEMultipart() - msg['to'] = secrets.admin_address - msg['from'] = secrets.from_address - msg['subject'] = 'Pending SweFreq requests' - msg['reply-to'] = secrets.reply_to_address - body = "There are pending requests for SweFreq accounts, please visit http://swefreq.nbis.se" + msg = MIMEMultipart() + msg['to'] = secrets.admin_address # pylint: disable=no-member + msg['from'] = secrets.from_address # pylint: disable=no-member + msg['subject'] = 'Pending SweFreq requests' + msg['reply-to'] = secrets.reply_to_address # pylint: disable=no-member + body = "There are pending requests for SweFreq accounts, please visit http://swefreq.nbis.se" msg.attach(MIMEText(body, 'plain')) - server = smtplib.SMTP(secrets.mail_server) + server = smtplib.SMTP(secrets.mail_server) # pylint: disable=no-member server.sendmail(msg['from'], [msg['to']], msg.as_string()) if __name__ == '__main__': requests = db.get_outstanding_requests(1) try: - requests.get() # There's at least one request + requests.get() # There's at least one request send_email() except peewee.DoesNotExist: # No new users so we don't send any emails. From be1a3871dc0ee3b8e50d470d422998335f28e3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 11:14:58 +0200 Subject: [PATCH 17/25] Pylint fixes in importer.py. --- scripts/importer/importer.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/importer/importer.py b/scripts/importer/importer.py index 28717925b..435a3103d 100755 --- a/scripts/importer/importer.py +++ b/scripts/importer/importer.py @@ -97,7 +97,8 @@ PARSER.add_argument("--add_mates", action="store_true", help=("Parse MANTA file and add the breakends to the db")) PARSER.add_argument("--add_reversed_mates", action="store_true", - help=("Assume input data only contain one line per BND, covering both directions")) + help=("Assume input data only contain one line per BND, " + + "covering both directions")) ARGS = PARSER.parse_args() @@ -107,9 +108,9 @@ if ARGS.add_reference: logging.info("Adding a new reference set using these sources:") - logging.info(" - Gencode: %s", ARGS.gencode_version) - logging.info(" - Ensembl: %s", ARGS.ensembl_version) - logging.info(" - dbNSFP: %s", ARGS.dbnsfp_version) + logging.info(f" - Gencode: {ARGS.gencode_version}") + logging.info(f" - Ensembl: {ARGS.ensembl_version}") + logging.info(f" - dbNSFP: {ARGS.dbnsfp_version}") IMPORTER = ReferenceSetImporter(ARGS) IMPORTER.prepare_data() @@ -118,7 +119,7 @@ IMPORTER.start_import() if ARGS.add_raw_data: - logging.info("Adding raw data %s", "(dry run)" if ARGS.dry_run else '') + logging.info(f"Adding raw data {'(dry run)' if ARGS.dry_run else ''}") IMPORTER = RawDataImporter(ARGS) IMPORTER.prepare_data() if not ARGS.disable_progress: From c0919b6ab4d3f417a25a6532e1f5deb5ae371fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 11:18:55 +0200 Subject: [PATCH 18/25] Pylint fixes in compile_template. --- scripts/compile_template.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/compile_template.py b/scripts/compile_template.py index 1955ac6da..49049c93e 100644 --- a/scripts/compile_template.py +++ b/scripts/compile_template.py @@ -1,37 +1,37 @@ -import jinja2 import argparse import datetime from pathlib import Path -class Jinja2Renderer(object): +import jinja2 + +class Jinja2Renderer: def __init__(self, directory): - self.env = jinja2.Environment( - autoescape = True, - block_start_string = '[%', - block_end_string = '%]', - variable_start_string = '[[', - variable_end_string = ']]', - comment_start_string = '[#', - comment_end_string = '#]', - - loader=jinja2.FileSystemLoader( str(directory) ) - ) + self.env = jinja2.Environment(autoescape=True, + block_start_string='[%', + block_end_string='%]', + variable_start_string='[[', + variable_end_string=']]', + comment_start_string='[#', + comment_end_string='#]', + loader=jinja2.FileSystemLoader(str(directory))) def render(self, file, **kwargs): - template = self.env.get_template( str(file) ) - return template.render( **kwargs ) + template = self.env.get_template(str(file)) + return template.render(**kwargs) def main(): parser = argparse.ArgumentParser() parser.add_argument("-b", "--basedir", dest='basedir', required=True) - parser.add_argument("-s", "--source", dest='src', required=True) + parser.add_argument("-s", "--source", dest='src', required=True) parser.add_argument("-d", "--develop", dest='develop', required=False, action='store_true') args = parser.parse_args() - src = Path(args.src).relative_to( args.basedir ) + src = Path(args.src).relative_to(args.basedir) renderer = Jinja2Renderer(args.basedir) - res = renderer.render( src, develop = args.develop, version=datetime.datetime.now().strftime("%Y%m%d%H%M%S.%f") ) + res = renderer.render(src, + develop=args.develop, + version=datetime.datetime.now().strftime("%Y%m%d%H%M%S.%f")) print(res) From 2813cd803873a7efa571979c8222fe51a9faf4cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 12:18:04 +0200 Subject: [PATCH 19/25] Disable relative-beyond-top-level. --- .pylintrc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 7b493e82f..11f764326 100644 --- a/.pylintrc +++ b/.pylintrc @@ -72,7 +72,8 @@ disable=too-few-public-methods, too-few-public-methods, keyword-arg-before-vararg, arguments-differ, - attribute-defined-outside-init + attribute-defined-outside-init, + relative-beyond-top-level # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option From 1dafcf68c5c0cbf95c0f4b237edbb59751233077 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 12:36:29 +0200 Subject: [PATCH 20/25] pylint fixes and disables in raw_data_importer. --- .../data_importer/raw_data_importer.py | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/scripts/importer/data_importer/raw_data_importer.py b/scripts/importer/data_importer/raw_data_importer.py index 2b3c5623e..351ddaff3 100644 --- a/scripts/importer/data_importer/raw_data_importer.py +++ b/scripts/importer/data_importer/raw_data_importer.py @@ -61,12 +61,12 @@ def _select_dataset_version(self): try: chosen_ds = db.Dataset.get(short_name=self.settings.dataset) except db.Dataset.DoesNotExist: - logging.error("Unknown dataset '%s'", self.settings.dataset) + logging.error(f"Unknown dataset {self.settings.dataset}") logging.info("Available datasets are:") for dataset in db.Dataset.select(): - logging.info(" * %s", dataset.short_name) + logging.info(f" * {dataset.short_name}") sys.exit(1) - logging.info("Using dataset {}".format(chosen_ds.short_name)) + logging.info(f"Using dataset {chosen_ds.short_name}") self.dataset = chosen_ds versions = [v for v in (db.DatasetVersion.select(). @@ -77,11 +77,11 @@ def _select_dataset_version(self): raise db.DatasetVersion.DoesNotExist("No versions exist for this dataset") if self.settings.version not in [v.version for v in versions]: - logging.error("Unknown version '%s' for dataset '%s'.", - self.settings.version, self.dataset.short_name) + logging.error("Unknown version '{self.settings.version}' " + + f"for dataset '{self.dataset.short_name}'.") logging.info("Available versions are:") for version in versions: - logging.info(" * %s", version.version) + logging.info(f" * {version.version}") sys.exit(1) self.dataset_version = [v for v in versions if v.version == self.settings.version][0] @@ -116,8 +116,8 @@ def _create_beacon_counts(self): datarow = {'datasetid': datasetid, 'callcount': self.counter['calls'], 'variantcount': self.counter['beaconvariants']} - logging.info('Dataset counts: callcount: %s, variantcount: %s', - datarow['callcount'], datarow['variantcount']) + logging.info(f"Dataset counts: callcount: {datarow['callcount']}, " + + f"variantcount: {datarow['variantcount']}") if not self.settings.dry_run: db.BeaconCounts.insert(datarow).execute() @@ -186,7 +186,7 @@ def _insert_coverage(self): finished=True) self.log_insertion(counter, "coverage", start) - def _parse_manta(self): + def _parse_manta(self): # pylint: disable=too-many-locals,too-many-branches,too-many-statements """Parse a manta file.""" header = [("chrom", str), ("pos", int), ("chrom_id", str), ("ref", str), ("alt", str)] @@ -211,7 +211,7 @@ def _parse_manta(self): base[header[i][0]] = header[i][1](item) elif i == 7: # only parse column 7 (maybe also for non-beacon-import?) - info = dict([(x.split('=', 1)) if '=' in x else (x, x) + info = dict([(x.split('=', 1)) if '=' in x else (x, x) # pylint: disable=consider-using-dict-comprehension for x in re.split(r';(?=\w)', item)]) if info.get('SVTYPE') != 'BND': @@ -230,7 +230,8 @@ def _parse_manta(self): for i, alt in enumerate(alt_alleles): data = dict(base) data['allele_freq'] = float(info.get('FRQ')) - data['alt'], data['mate_chrom'], data['mate_start'] = re.search(r'(.+)[[\]](.*?):(\d+)[[\]]', alt).groups() + data['alt'], data['mate_chrom'], data['mate_start'] = \ + re.search(r'(.+)[[\]](.*?):(\d+)[[\]]', alt).groups() if data['mate_chrom'].startswith('GL') or data['mate_chrom'].startswith('MT'): # A BND from a chromosome to GL or MT. # TODO ask a bioinformatician if these cases should be included or not @@ -244,7 +245,7 @@ def _parse_manta(self): # Set to 0 rather than None, as the type should be int (according to the Beacon # API specificition). data['allele_count'] = info.get('AC', 0) - data['allele_num'] = info.get('AN',0) + data['allele_num'] = info.get('AN', 0) batch += [data] if self.settings.add_reversed_mates: @@ -297,9 +298,9 @@ def _parse_manta(self): finished=True) self.log_insertion(counter, "breakend", start) - def _insert_variants(self): + def _insert_variants(self): # pylint: disable=too-many-locals,too-many-branches,too-many-statements """Import variants from a VCF file.""" - logging.info("Inserting variants%s", " (dry run)" if self.settings.dry_run else "") + logging.info(f"Inserting variants{' (dry run)' if self.settings.dry_run else ''}") header = [("chrom", str), ("pos", int), ("rsid", str), ("ref", str), ("alt", str), ("site_quality", float), ("filter_string", str)] start = time.time() @@ -312,7 +313,7 @@ def _insert_variants(self): samples = 0 vep_field_names = None with db.database.atomic(): - for filename in self.settings.variant_file: + for filename in self.settings.variant_file: # pylint: disable=too-many-nested-blocks # Get reference set for the variant ref_set = self.dataset_version.reference_set @@ -338,7 +339,8 @@ def _insert_variants(self): if not self.settings.beacon_only: if vep_field_names is None: - logging.error("VEP_field_names is empty. Make sure VCF header is present.") + logging.error("VEP_field_names is empty. " + + "Make sure VCF header is present.") sys.exit(1) base = {} @@ -349,7 +351,7 @@ def _insert_variants(self): base[header[i][0]] = header[i][1](item) elif i == 7 or not self.settings.beacon_only: # only parse column 7 (maybe also for non-beacon-import?) - info = dict([(x.split('=', 1)) if '=' in x else (x, x) + info = dict([(x.split('=', 1)) if '=' in x else (x, x) # pylint: disable=consider-using-dict-comprehension for x in re.split(r';(?=\w)', item)]) if base["chrom"].startswith('GL') or base["chrom"].startswith('MT'): @@ -424,7 +426,7 @@ def _insert_variants(self): data['pos'], data['ref'], data['alt']) - data['quality_metrics'] = dict([(x, info[x]) for x in METRICS if x in info]) + data['quality_metrics'] = {x: info[x] for x in METRICS if x in info} batch += [data] if self.settings.count_calls: self.get_callcount(data) # count calls (one per reference) @@ -456,7 +458,8 @@ def _insert_variants(self): indexes = [] for entry in batch: indexes.append(db.Variant.select(db.Variant.id) - .where(db.Variant.variant_id == entry['variant_id']) + .where(db.Variant.variant_id == \ + entry['variant_id']) .get().id) self.add_variant_genes(indexes, genes, ref_genes) self.add_variant_transcripts(indexes, From 9fd2fcae6ff56dee9fff4be840f60cfc8c4fde26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 12:56:31 +0200 Subject: [PATCH 21/25] pylint fixes and disables in reference_set_importer. --- .../data_importer/reference_set_importer.py | 51 ++++++++++++------- 1 file changed, 33 insertions(+), 18 deletions(-) diff --git a/scripts/importer/data_importer/reference_set_importer.py b/scripts/importer/data_importer/reference_set_importer.py index ca3ba34aa..675b6c353 100644 --- a/scripts/importer/data_importer/reference_set_importer.py +++ b/scripts/importer/data_importer/reference_set_importer.py @@ -12,7 +12,7 @@ import db from .data_importer import DataImporter -class ReferenceSetImporter(DataImporter): +class ReferenceSetImporter(DataImporter): # pylint: disable=too-many-instance-attributes """Import a reference set into db.""" GENCODE = ("ftp://ftp.sanger.ac.uk/pub/gencode/Gencode_human/" + @@ -58,8 +58,10 @@ def _insert_features(self): start = time.time() last_progress = -1 batch = [] + i = 0 with db.database.atomic(): - for i, feature in enumerate(self.features): + i = 0 + for feature in self.features: batch += [{'gene':self.gene_db_ids[feature['gene_id']], 'transcript':self.transcript_db_ids[feature['transcript_id']], 'chrom':feature['chrom'], @@ -74,9 +76,12 @@ def _insert_features(self): batch = [] last_progress = self._update_progress_bar(i, len(self.features), last_progress) + i += 1 + if batch: if not self.settings.dry_run: db.Feature.insert_many(batch).execute() + last_progress = self._update_progress_bar(i, len(self.features), last_progress, finished=True) @@ -87,7 +92,8 @@ def _insert_genes(self): logging.info("Inserting genes into database") start = time.time() last_progress = -1 - for i, gene in enumerate(self.genes): + i = 0 + for gene in self.genes: # As far as I know I can't batch insert these and still get the id's back db_gene = db.Gene(reference_set=self.db_reference, gene_id=gene['gene_id'], @@ -113,6 +119,8 @@ def _insert_genes(self): pass last_progress = self._update_progress_bar(i, len(self.genes), last_progress) + i += 1 + last_progress = self._update_progress_bar(i, len(self.genes), last_progress, finished=True) logging.info(f"Genes inserted in {self._time_since(start)}") @@ -129,7 +137,7 @@ def _insert_reference(self): self.db_reference.id = max_id.id + 1 else: self.db_reference.save() - logging.info("Reference %s created", self.db_reference.id) + logging.info(f"Reference {self.db_reference.id} created") def _insert_transcripts(self): """Insert trabscripts into db.""" @@ -137,23 +145,26 @@ def _insert_transcripts(self): start = time.time() last_progress = -1 - for i, transcript in enumerate(self.transcripts): - db_transcript = db.Transcript(transcript_id=transcript['transcript_id'], - gene=self.gene_db_ids[transcript['gene_id']], - mim_annotation=transcript.get('mim_annotation', None), - mim_gene_accession=transcript.get('mim_gene_accession', None), - chrom=transcript['chrom'], - start=transcript['start'], - stop=transcript['stop'], - strand=transcript['strand']) + i = 0 + for transcript in self.transcripts: + db_trans = db.Transcript(transcript_id=transcript['transcript_id'], + gene=self.gene_db_ids[transcript['gene_id']], + mim_annotation=transcript.get('mim_annotation', None), + mim_gene_accession=transcript.get('mim_gene_accession', None), + chrom=transcript['chrom'], + start=transcript['start'], + stop=transcript['stop'], + strand=transcript['strand']) if self.settings.dry_run: self.transcript_db_ids[transcript['transcript_id']] = 0 else: - db_transcript.save() - self.transcript_db_ids[transcript['transcript_id']] = db_transcript.id + db_trans.save() + self.transcript_db_ids[transcript['transcript_id']] = db_trans.id last_progress = self._update_progress_bar(i, len(self.transcripts), last_progress) + i += 1 + last_progress = self._update_progress_bar(i, len(self.transcripts), last_progress, finished=True) @@ -202,7 +213,8 @@ def _open_dbnsfp(self): def _open_ensembl(self): """Connect to the given ensembl database.""" logging.info("----- Opening ensembl database connection -----") - self.ensembl = self._connect(*(ReferenceSetImporter.ENSEMBL + (self.settings.ensembl_version,))) + self.ensembl = self._connect(*(ReferenceSetImporter.ENSEMBL + + (self.settings.ensembl_version,))) def _open_gencode(self): """Download (if needed) and opens the given gencode file.""" @@ -264,13 +276,16 @@ def _read_ensembl(self): canonical_dict[transcript[0]] = transcript[1] last_progress = -1.0 - for i, gene in enumerate(self.genes): + i = 0 + for gene in self.genes: if gene['gene_id'] in canonical_dict: self.genes[i]['canonical_transcript'] = canonical_dict[gene['gene_id']] self.counters['genes'] += 1 if self.numbers['genes'] is not None: last_progress = self._update_progress_bar(i, self.numbers['genes'], last_progress) + i += 1 + if self.numbers['genes'] is not None: last_progress = self._update_progress_bar(i, self.numbers['genes'], last_progress, finished=True) @@ -364,7 +379,7 @@ def start_import(self): self.counters['features'] += 1 continue - except Exception as error: + except Exception as error: # pylint: disable=broad-except logging.error("{}".format(error)) break if self.numbers['genes'] is not None: From 990bc698cdba431e90dd38fe1b2735adedb63e0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 13:10:14 +0200 Subject: [PATCH 22/25] pylint fixes and disables in data_importer. --- scripts/importer/data_importer/data_importer.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/importer/data_importer/data_importer.py b/scripts/importer/data_importer/data_importer.py index 4d61dfbaf..85fb8ea48 100644 --- a/scripts/importer/data_importer/data_importer.py +++ b/scripts/importer/data_importer/data_importer.py @@ -26,7 +26,7 @@ def __init__(self, settings): self.progress_bar = not settings.disable_progress self.in_file = None - def _connect(self, host, user, passwd, database): + def _connect(self, host, user, passwd, database): # pylint: disable=no-self-use try: logging.info("Connecting to database {}".format(database)) database = MySQLdb.connect(host=host, @@ -34,7 +34,7 @@ def _connect(self, host, user, passwd, database): passwd=passwd, db=database) return database.cursor() - except MySQLdb.Error as error: + except MySQLdb.Error as error: # pylint: disable=no-member logging.error("Error connecting: {}".format(error)) def _download(self, base_url, version=None): @@ -87,16 +87,18 @@ def _download_and_open(self, base_url, version=None): filename = self._download(base_url, version) return self._open(filename) - def _open(self, filename, binary=True): + def _open(self, filename, binary=True): # pylint: disable=no-self-use mode = 'rb' if binary else 'rt' encoding = None if binary else 'utf8' try: logging.debug("Opening file {}".format(filename)) - return gzip.open(filename, mode, encoding=encoding) if filename.endswith(".gz") else open(filename) + return gzip.open(filename, mode, encoding=encoding) \ + if filename.endswith(".gz") \ + else open(filename) except IOError as error: logging.error("IOERROR: {}".format(error)) - def _time_format(self, seconds): + def _time_format(self, seconds): # pylint: disable=no-self-use hour, rem = divmod(seconds, 3600) mins, secs = divmod(rem, 60) retval = "" @@ -113,7 +115,7 @@ def _time_since(self, start): def _time_to(self, start, progress=0.01): return self._time_format((time.time() - start)/progress) - def _update_progress_bar(self, current_count, total, last_progress, finished=False): + def _update_progress_bar(self, current_count, total, last_progress, finished=False): # pylint: disable=no-self-use if not finished: progress = current_count/total else: From 09bdb43b6a7143939037ab57f290aa20fa6888c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 13:11:57 +0200 Subject: [PATCH 23/25] Disable fixme warning in raw_data_importer. --- scripts/importer/data_importer/raw_data_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/importer/data_importer/raw_data_importer.py b/scripts/importer/data_importer/raw_data_importer.py index 351ddaff3..1451f8096 100644 --- a/scripts/importer/data_importer/raw_data_importer.py +++ b/scripts/importer/data_importer/raw_data_importer.py @@ -234,7 +234,7 @@ def _parse_manta(self): # pylint: disable=too-many-locals,too-many-branches,too re.search(r'(.+)[[\]](.*?):(\d+)[[\]]', alt).groups() if data['mate_chrom'].startswith('GL') or data['mate_chrom'].startswith('MT'): # A BND from a chromosome to GL or MT. - # TODO ask a bioinformatician if these cases should be included or not + # TODO ask a bioinformatician if these cases should be included or not # pylint: disable=fixme continue data['mate_id'] = info.get('MATEID', '') data['variant_id'] = '{}-{}-{}-{}'.format(data['chrom'], From f98eaab8d8da58a14279d88c94a8c0a4ff8935fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 13:24:21 +0200 Subject: [PATCH 24/25] Run pylint on scripts/ as well. --- test/travis_script.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/travis_script.sh b/test/travis_script.sh index 225b03d05..017a8f7ea 100755 --- a/test/travis_script.sh +++ b/test/travis_script.sh @@ -171,6 +171,8 @@ RETURN_VALUE=$((RETURN_VALUE + $?)) echo '>>> Code evaluation' pylint backend RETURN_VALUE=$((RETURN_VALUE + $?)) +pylint scripts +RETURN_VALUE=$((RETURN_VALUE + $?)) echo '>>> Finalising: Combine coverage' From b78f78672bb90d13cf991c581501c6a3d7820b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linus=20=C3=96stberg?= Date: Thu, 15 Aug 2019 13:42:24 +0200 Subject: [PATCH 25/25] Should be f-string. --- scripts/importer/data_importer/raw_data_importer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/importer/data_importer/raw_data_importer.py b/scripts/importer/data_importer/raw_data_importer.py index 1451f8096..dccb06b4d 100644 --- a/scripts/importer/data_importer/raw_data_importer.py +++ b/scripts/importer/data_importer/raw_data_importer.py @@ -77,7 +77,7 @@ def _select_dataset_version(self): raise db.DatasetVersion.DoesNotExist("No versions exist for this dataset") if self.settings.version not in [v.version for v in versions]: - logging.error("Unknown version '{self.settings.version}' " + + logging.error(f"Unknown version '{self.settings.version}' " + f"for dataset '{self.dataset.short_name}'.") logging.info("Available versions are:") for version in versions: