From 4d34b74108cfdcb66515e266af448fa16f566ebc Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 10:57:47 +0200 Subject: [PATCH 01/25] Flake8 correction of ingest.py --- lega/ingest.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lega/ingest.py b/lega/ingest.py index af396402..1b7b2f6d 100644 --- a/lega/ingest.py +++ b/lega/ingest.py @@ -26,7 +26,7 @@ from legacryptor.crypt4gh import get_header from .conf import CONF -from .utils import db, exceptions, checksum, sanitize_user_id, storage +from .utils import db, exceptions, sanitize_user_id, storage from .utils.amqp import consume, publish, get_connection LOG = logging.getLogger(__name__) @@ -49,7 +49,7 @@ def work(fs, channel, data): # Insert in database file_id = db.insert_file(filepath, user_id) - data['file_id'] = file_id # must be there: database error uses it + data['file_id'] = file_id # must be there: database error uses it # Find inbox inbox = Path(CONF.get_value('inbox', 'location', raw=True) % user_id) @@ -59,7 +59,7 @@ def work(fs, channel, data): inbox_filepath = inbox / filepath.lstrip('/') LOG.info(f"Inbox file path: {inbox_filepath}") if not inbox_filepath.exists(): - raise exceptions.NotFoundInInbox(filepath) # return early + raise exceptions.NotFoundInInbox(filepath) # return early # Ok, we have the file in the inbox @@ -71,7 +71,7 @@ def work(fs, channel, data): LOG.debug(f'Sending message to CentralEGA: {data}') publish(org_msg, channel, 'cega', 'files.processing') org_msg.pop('status', None) - + # Strip the header out and copy the rest of the file to the vault LOG.debug(f'Opening {inbox_filepath}') with open(inbox_filepath, 'rb') as infile: @@ -80,11 +80,11 @@ def work(fs, channel, data): header_hex = (beginning+header).hex() data['header'] = header_hex - db.store_header(file_id, header_hex) # header bytes will be .hex() + db.store_header(file_id, header_hex) # header bytes will be .hex() target = fs.location(file_id) LOG.info(f'[{fs.__class__.__name__}] Moving the rest of {filepath} to {target}') - target_size = fs.copy(infile, target) # It will copy the rest only + target_size = fs.copy(infile, target) # It will copy the rest only LOG.info(f'Vault copying completed. Updating database') db.set_archived(file_id, target, target_size) @@ -97,7 +97,7 @@ def main(args=None): if not args: args = sys.argv[1:] - CONF.setup(args) # re-conf + CONF.setup(args) # re-conf fs = getattr(storage, CONF.get_value('vault', 'driver', default='FileStorage')) broker = get_connection('broker') @@ -106,5 +106,6 @@ def main(args=None): # upstream link configured in local broker consume(do_work, broker, 'files', 'archived') + if __name__ == '__main__': main() From 70385c78969b3545cd9555869b0eacd6531c6f83 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 11:06:14 +0200 Subject: [PATCH 02/25] Flake8 correction of keyserver.py --- lega/keyserver.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/lega/keyserver.py b/lega/keyserver.py index 8d85f911..3fa9b18c 100644 --- a/lega/keyserver.py +++ b/lega/keyserver.py @@ -94,12 +94,11 @@ def _check_limit(self): def clear(self): """Clear all cache.""" - #self.store = dict() self.store.clear() -_cache = None # key IDs are uppercase -_active = None # will be a KeyID (not a key name) +_cache = None # key IDs are uppercase +_active = None # will be a KeyID (not a key name) #################################### # Caching the keys @@ -127,13 +126,13 @@ async def retrieve_active_key(request): key_type = request.match_info['key_type'].lower() LOG.debug(f'Requesting active ({key_type}) key') if key_type not in ('public', 'private'): - return web.HTTPForbidden() # web.HTTPBadRequest() + return web.HTTPForbidden() # web.HTTPBadRequest() key_format = 'armored' if request.content_type == 'text/plain' else None if _active is None: return web.HTTPNotFound() k = _cache.get(_active, key_type, key_format=key_format) if k: - return web.Response(body=k) # web.Response(text=k.hex()) + return web.Response(body=k) # web.Response(text=k.hex()) else: LOG.warn(f"Requested active ({key_type}) key not found.") return web.HTTPNotFound() @@ -144,13 +143,13 @@ async def retrieve_key(request): requested_id = request.match_info['requested_id'] key_type = request.match_info['key_type'].lower() if key_type not in ('public', 'private'): - return web.HTTPForbidden() # web.HTTPBadRequest() + return web.HTTPForbidden() # web.HTTPBadRequest() key_id = requested_id[-16:].upper() key_format = 'armored' if request.content_type == 'text/plain' else None LOG.debug(f'Requested {key_type.upper()} key with ID {requested_id}') k = _cache.get(key_id, key_type, key_format=key_format) if k: - return web.Response(body=k) # web.Response(text=value.hex()) + return web.Response(body=k) # web.Response(text=value.hex()) else: LOG.warn(f"Requested key {requested_id} not found.") return web.HTTPNotFound() @@ -181,7 +180,7 @@ async def healthcheck(request): @routes.get('/admin/ttl') async def check_ttl(request): """Evict from the cache if TTL expired - and return the keys that survived""" # ehh...why? /Fred + and return the keys that survived""" # ehh...why? /Fred LOG.debug('Admin TTL') expire = _cache.check_ttl() if expire: @@ -196,7 +195,8 @@ def load_keys_conf(store): _cache = Cache() # Load all the keys in the store for section in store.sections(): - _unlock_key(section, **dict(store.items(section))) # includes defaults + _unlock_key(section, **dict(store.items(section))) # includes defaults + alive = True # used to set if the keyserver is alive in the shutdown @@ -239,8 +239,8 @@ def main(args=None): host = CONF.get_value('keyserver', 'host') # fallbacks are in defaults.ini port = CONF.get_value('keyserver', 'port', conv=int) - health_check_url='http://{}:{}{}'.format(host, port, CONF.get_value('keyserver', 'health_endpoint')) - status_check_url='http://{}:{}{}'.format(host, port, CONF.get_value('keyserver', 'status_endpoint')) + health_check_url = 'http://{}:{}{}'.format(host, port, CONF.get_value('keyserver', 'health_endpoint')) + status_check_url = 'http://{}:{}{}'.format(host, port, CONF.get_value('keyserver', 'status_endpoint')) eureka_endpoint = CONF.get_value('eureka', 'endpoint') From 5b3b844d15ded1ddc9778db5702c7b5b625cc70c Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 11:07:37 +0200 Subject: [PATCH 03/25] Flake8 correction of mapper.py --- lega/mapper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lega/mapper.py b/lega/mapper.py index 7bfcc4e9..ce3b1c4e 100644 --- a/lega/mapper.py +++ b/lega/mapper.py @@ -39,12 +39,13 @@ def main(args=None): if not args: args = sys.argv[1:] - CONF.setup(args) # re-conf + CONF.setup(args) # re-conf broker = get_connection('broker') # upstream link configured in local broker consume(work, broker, 'stableIDs', None) + if __name__ == '__main__': main() From 15af4dd135447c0f8faca38fa264f9f7c3b9d89e Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 11:16:24 +0200 Subject: [PATCH 04/25] Flake8 correction of notifications.py --- lega/notifications.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/lega/notifications.py b/lega/notifications.py index bbed2fa6..cce0c853 100644 --- a/lega/notifications.py +++ b/lega/notifications.py @@ -12,16 +12,18 @@ import os import asyncio import uvloop + +from .conf import CONF +from .utils.amqp import get_connection, publish +from .utils.checksum import calculate + + asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) host = '127.0.0.1' port = 8888 delim = b'$' -from .conf import CONF -from .utils.amqp import get_connection, publish -from .utils.checksum import calculate, supported_algorithms - LOG = logging.getLogger(__name__) class Forwarder(asyncio.Protocol): @@ -53,9 +55,9 @@ def parse(self, data): # We have 2 bars pos1 = data.find(delim) username = data[:pos1] - pos2 = data.find(delim,pos1+1) + pos2 = data.find(delim, pos1+1) filename = data[pos1+1:pos2] - yield (username.decode(),filename.decode()) + yield (username.decode(), filename.decode()) data = data[pos2+1:] def data_received(self, data): @@ -70,13 +72,15 @@ def data_received(self, data): def send_message(self, username, filename): inbox = self.inbox_location % username - filepath, filename = (os.path.join(inbox, filename.lstrip('/')), filename) if self.isolation \ - else (filename, filename[len(inbox):]) # surely there is better! + filepath, filename = (filename, filename[len(inbox):]) + if self.isolation: + filepath, filename = (os.path.join(inbox, filename.lstrip('/')), filename) + LOG.debug("Filepath %s", filepath) - msg = { 'user': username, - 'filepath': filename, - 'filesize': os.stat(filepath).st_size - } + msg = {'user': username, + 'filepath': filename, + 'filesize': os.stat(filepath).st_size + } c = calculate(filepath, 'sha256') if c: msg['encrypted_integrity'] = {'algorithm': 'sha256', 'checksum': c} From 83a4bbac34bb1464d1e13c0b2d7db9e0d88484d3 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 11:18:12 +0200 Subject: [PATCH 05/25] Flake8 correction of verify.py --- lega/verify.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lega/verify.py b/lega/verify.py index 4d344152..2edd5d0e 100644 --- a/lega/verify.py +++ b/lega/verify.py @@ -22,7 +22,7 @@ from .conf import CONF from .utils import db, exceptions, storage -from .utils.amqp import consume, publish, get_connection +from .utils.amqp import consume, get_connection LOG = logging.getLogger(__name__) @@ -34,8 +34,8 @@ def get_records(header): LOG.info(f'Retrieving the Private Key from {keyurl} (verify certificate: {verify})') if verify: - ctx=None # nothing to be done: done by default in urlopen - else: # no verification + ctx = None # nothing to be done: done by default in urlopen + else: # no verification import ssl ctx = ssl.create_default_context() ctx.check_hostname = False @@ -48,7 +48,7 @@ def get_records(header): except HTTPError as e: LOG.error(e) msg = str(e) - if e.code == 404: # If key not found, then probably wrong key. + if e.code == 404: # If key not found, then probably wrong key. raise exceptions.PGPKeyError(msg) # Otherwise raise exceptions.KeyserverError(msg) @@ -63,18 +63,19 @@ def work(chunk_size, mover, channel, data): LOG.info('Verification | message: %s', data) file_id = data['file_id'] - header = bytes.fromhex(data['header'])[16:] # in hex -> bytes, and take away 16 bytes + header = bytes.fromhex(data['header'])[16:] # in hex -> bytes, and take away 16 bytes vault_path = data['vault_path'] # Get it from the header and the keyserver - records, key_id = get_records(header) # might raise exception - r = records[0] # only first one + records, key_id = get_records(header) # might raise exception + r = records[0] # only first one LOG.info('Opening vault file: %s', vault_path) # If you can decrypt... the checksum is valid # Calculate the checksum of the original content md = hashlib.sha256() + def checksum_content(data): md.update(data) @@ -92,7 +93,7 @@ def checksum_content(data): org_msg = data['org_msg'] org_msg.pop('file_id', None) org_msg['reference'] = file_id - org_msg['checksum'] = { 'value': digest, 'algorithm': 'sha256' } + org_msg['checksum'] = {'value': digest, 'algorithm': 'sha256'} LOG.debug(f"Reply message: {org_msg}") return org_msg @@ -101,15 +102,16 @@ def main(args=None): if not args: args = sys.argv[1:] - CONF.setup(args) # re-conf + CONF.setup(args) # re-conf store = getattr(storage, CONF.get_value('vault', 'driver', default='FileStorage')) - chunk_size = CONF.get_value('vault', 'chunk_size', conv=int, default=1<<22) # 4 MB + chunk_size = CONF.get_value('vault', 'chunk_size', conv=int, default=1 << 22) # 4 MB broker = get_connection('broker') do_work = partial(work, chunk_size, store(), broker.channel()) consume(do_work, broker, 'archived', 'completed') + if __name__ == '__main__': main() From f0c6e55b9c1b6e8435f1cf1e3f3b9d7679253cff Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 11:26:23 +0200 Subject: [PATCH 06/25] Flake8 correction of __init__.py --- lega/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lega/__init__.py b/lega/__init__.py index 48480b6d..a9cd88b1 100644 --- a/lega/__init__.py +++ b/lega/__init__.py @@ -6,7 +6,7 @@ __title__ = 'Local EGA' __version__ = VERSION = '1.1' __author__ = 'Frédéric Haziza' -#__license__ = 'Apache 2.0' +__license__ = 'Apache 2.0' __copyright__ = 'Local EGA @ NBIS Sweden' # Set default logging handler to avoid "No handler found" warnings. From be7d8cf2ddf59c1dbc4c19b738c6a6721752892d Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 12:27:15 +0200 Subject: [PATCH 07/25] Flake8 correction of conf/__init__.py --- lega/conf/__init__.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lega/conf/__init__.py b/lega/conf/__init__.py index a35d2170..1f4a53ef 100644 --- a/lega/conf/__init__.py +++ b/lega/conf/__init__.py @@ -22,9 +22,7 @@ import sys import os import configparser -import logging from logging.config import fileConfig, dictConfig -import lega.utils.logging from pathlib import Path import yaml from hashlib import md5 @@ -138,9 +136,8 @@ def get_value(self, section, option, conv=str, default=None, raw=False): ``section`` and ``option`` are mandatory while ``conv``, ``default`` (fallback) and ``raw`` are optional. """ result = os.environ.get(f'{section.upper()}_{option.upper()}', None) - if result is not None: # it might be empty + if result is not None: # it might be empty return self._convert(result, conv) - #if self.has_option(section, option): return self._convert(self.get(section, option, fallback=default, raw=raw), conv) def _convert(self, value, conv): @@ -155,7 +152,7 @@ def _convert(self, value, conv): else: raise ValueError(f"Invalid truth value: {val}") else: - return conv(value) # raise error in case we can't convert an empty value + return conv(value) # raise error in case we can't convert an empty value CONF = Configuration() From 539da23f4485cf42560b5eff1220a8a41129e3d9 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 12:28:11 +0200 Subject: [PATCH 08/25] Flake8 correction of conf/__main__.py --- lega/conf/__main__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lega/conf/__main__.py b/lega/conf/__main__.py index 3162e1a0..ea0a9744 100644 --- a/lega/conf/__main__.py +++ b/lega/conf/__main__.py @@ -15,12 +15,12 @@ def main(args=None): parser = argparse.ArgumentParser(description="Forward message between CentralEGA's broker and the local one", allow_abbrev=False) parser.add_argument('--conf', help='configuration file, in INI or YAML format') - parser.add_argument('--log', help='configuration file for the loggers') - + parser.add_argument('--log', help='configuration file for the loggers') + parser.add_argument('--list', dest='list_content', action='store_true', help='Lists the content of the configuration file') pargs = parser.parse_args(args) - - CONF.setup( args ) + + CONF.setup(args) print(repr(CONF)) From ad0895f357cacd863891bb3cb0e30561d2992b5f Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 12:29:02 +0200 Subject: [PATCH 09/25] Flake8 correction of utils/__init__.py --- lega/utils/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lega/utils/__init__.py b/lega/utils/__init__.py index 3419a361..9884e211 100644 --- a/lega/utils/__init__.py +++ b/lega/utils/__init__.py @@ -8,7 +8,7 @@ def get_file_content(f, mode='rb'): try: - with open( f, mode) as h: + with open(f, mode) as h: return h.read() except OSError as e: LOG.error(f'Error reading {f}: {e!r}') @@ -18,4 +18,3 @@ def sanitize_user_id(user): '''Returns username without host part of an ID on the form name@something''' return user.split('@')[0] - From bf9e137c77b8f5968053d3e451195752c5ebd93e Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 12:32:18 +0200 Subject: [PATCH 10/25] Flake8 correction of utils/amqp.py --- lega/utils/amqp.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lega/utils/amqp.py b/lega/utils/amqp.py index a6b88865..68df89bb 100644 --- a/lega/utils/amqp.py +++ b/lega/utils/amqp.py @@ -31,7 +31,7 @@ def get_connection(domain, blocking=True): CONF.get_value(domain, 'password', default='guest') ), 'connection_attempts': CONF.get_value(domain, 'connection_attempts', conv=int, default=10), - 'retry_delay': CONF.get_value(domain,'retry_delay', conv=int, default=10), # seconds + 'retry_delay': CONF.get_value(domain, 'retry_delay', conv=int, default=10), # seconds } heartbeat = CONF.get_value(domain, 'heartbeat', conv=int, default=0) if heartbeat is not None: # can be 0 @@ -46,7 +46,7 @@ def get_connection(domain, blocking=True): params['ssl_options'] = { 'ca_certs': CONF.get_value(domain, 'cacert'), 'certfile': CONF.get_value(domain, 'cert'), - 'keyfile': CONF.get_value(domain, 'keyfile'), + 'keyfile': CONF.get_value(domain, 'keyfile'), 'cert_reqs': 2, # ssl.CERT_REQUIRED is actually } @@ -62,12 +62,12 @@ def publish(message, channel, exchange, routing, correlation_id=None): Sending a message to the local broker with ``path`` was updated ''' LOG.debug(f'Sending {message} to exchange: {exchange} [routing key: {routing}]') - channel.basic_publish(exchange = exchange, - routing_key = routing, - body = json.dumps(message), - properties = pika.BasicProperties(correlation_id=correlation_id or str(uuid.uuid4()), - content_type='application/json', - delivery_mode=2)) + channel.basic_publish(exchange=exchange, + routing_key=routing, + body=json.dumps(message), + properties=pika.BasicProperties(correlation_id=correlation_id or str(uuid.uuid4()), + content_type='application/json', + delivery_mode=2)) def consume(work, connection, from_queue, to_routing): @@ -84,12 +84,12 @@ def consume(work, connection, from_queue, to_routing): routing key. ''' - assert( from_queue ) + assert(from_queue) LOG.debug(f'Consuming message from {from_queue}') from_channel = connection.channel() - from_channel.basic_qos(prefetch_count=1) # One job per worker + from_channel.basic_qos(prefetch_count=1) # One job per worker to_channel = connection.channel() def process_request(channel, method_frame, props, body): @@ -98,12 +98,12 @@ def process_request(channel, method_frame, props, body): LOG.debug(f'Consuming message {message_id} (Correlation ID: {correlation_id})') # Process message in JSON format - answer = work( json.loads(body) ) # Exceptions should be already caught + answer = work(json.loads(body)) # Exceptions should be already caught # Publish the answer if answer: - assert( to_routing ) - publish(answer, to_channel, 'lega', to_routing, correlation_id = props.correlation_id) + assert(to_routing) + publish(answer, to_channel, 'lega', to_routing, correlation_id=props.correlation_id) # Acknowledgment: Cancel the message resend in case MQ crashes LOG.debug(f'Sending ACK for message {message_id} (Correlation ID: {correlation_id})') From 79dc733bbb780532a4cd6a9e0b1fa6f24908ffef Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 12:33:12 +0200 Subject: [PATCH 11/25] Flake8 correction of utils/checksum.py --- lega/utils/checksum.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lega/utils/checksum.py b/lega/utils/checksum.py index c6f048bf..8a05e9c6 100644 --- a/lega/utils/checksum.py +++ b/lega/utils/checksum.py @@ -31,7 +31,7 @@ def calculate(filepath, algo, bsize=8192): ''' try: m = instantiate(algo) - with open(filepath, 'rb') as f: # Open the file in binary mode. No encoding dance. + with open(filepath, 'rb') as f: # Open the file in binary mode. No encoding dance. while True: data = f.read(bsize) if not data: @@ -43,10 +43,10 @@ def calculate(filepath, algo, bsize=8192): return None -def is_valid(filepath, digest, hashAlgo = 'md5'): +def is_valid(filepath, digest, hashAlgo='md5'): '''Verify the integrity of a file against a hash value.''' - assert( isinstance(digest,str) ) + assert(isinstance(digest, str)) res = calculate(filepath, hashAlgo) LOG.debug('Calculated digest: '+res) @@ -67,10 +67,9 @@ def get_from_companion(filepath): try: with open(companion, 'rt', encoding='utf-8') as f: return f.read(), h - except OSError as e: # Not found, not readable, ... + except OSError as e: # Not found, not readable, ... LOG.debug(f'Companion {companion}: {e!r}') # Check the next - else: # no break statement was encountered + else: # no break statement was encountered raise CompanionNotFound(filepath) - From ce80de3c296ebbd4a446506aa7e040dccc24381a Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 12:39:48 +0200 Subject: [PATCH 12/25] Flake8 correction of utils/eureka.py --- lega/utils/eureka.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lega/utils/eureka.py b/lega/utils/eureka.py index fff839d8..25d85fcb 100644 --- a/lega/utils/eureka.py +++ b/lega/utils/eureka.py @@ -27,6 +27,7 @@ def retry_loop(func): """Decorator retry something ``try`` times every ``try_interval`` seconds.""" assert asyncio.iscoroutinefunction(func), "This decorator is only for coroutines" + @wraps(func) async def wrapper(*args, **kwargs): """Main retry loop.""" From 3de80b7f39d007858d0430d1fe71d6d9898a87c7 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 12:43:41 +0200 Subject: [PATCH 13/25] Flake8 correction of utils/exceptions.py --- lega/utils/exceptions.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/lega/utils/exceptions.py b/lega/utils/exceptions.py index f10c15da..5c61b6ab 100644 --- a/lega/utils/exceptions.py +++ b/lega/utils/exceptions.py @@ -3,47 +3,53 @@ Exceptions ''' -import legacryptor - ############################################################################# # User Errors ############################################################################# class FromUser(Exception): - def __str__(self): # Informal description + def __str__(self): # Informal description return 'Incorrect user input' - def __repr__(self): # Technical description + + def __repr__(self): # Technical description return str(self) class NotFoundInInbox(FromUser): def __init__(self, filename): self.filename = filename + def __str__(self): return f'File not found in inbox' + def __repr__(self): return f'Inbox missing file: {self.filename}' class UnsupportedHashAlgorithm(FromUser): def __init__(self, algo): self.algo = algo + def __str__(self): return f'Unsupported hash algorithm: {self.algo!r}' class CompanionNotFound(FromUser): def __init__(self, name): self.name = name + def __str__(self): return f'Companion file not found in inbox' + def __repr__(self): return f'Companion file not found for {self.name}' - + class Checksum(FromUser): def __init__(self, algo, file=None, decrypted=False): self.algo = algo self.decrypted = decrypted self.file = file + def __str__(self): return 'Invalid {} checksum for the {} file'.format(self.algo, 'original' if self.decrypted else 'encrypted') + def __repr__(self): return 'Invalid {} checksum for the {} file: {}'.format(self.algo, 'original' if self.decrypted else 'encrypted', self.file) @@ -54,16 +60,20 @@ def __repr__(self): class PGPKeyError(Exception): def __init__(self, msg): self.msg = msg + def __str__(self): return 'PGP Key error' + def __repr__(self): return f'PGP Key error: {self.msg}' class KeyserverError(Exception): def __init__(self, msg): self.msg = msg + def __str__(self): return 'Keyserver error' + def __repr__(self): return f'Keyserver error: {self.msg}' @@ -77,8 +87,9 @@ def __init__(self, user, filename, enc_checksum_hash, enc_checksum_algorithm): self.filename = filename self.enc_checksum_hash = enc_checksum_hash self.enc_checksum_algorithm = enc_checksum_algorithm + def __repr__(self): return (f'Warning: File already processed\n' f'\t* user: {self.user}\n' f'\t* name: {self.filename}\n' - f'\t* Encrypted checksum: {enc_checksum_hash} (algorithm: {enc_checksum_algorithm}') + f'\t* Encrypted checksum: {self.enc_checksum_hash} (algorithm: {self.enc_checksum_algorithm}') From bb3eaf759773e414f50cca0cc6f40d31a0285dab Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 12:44:34 +0200 Subject: [PATCH 14/25] Flake8 correction of utils/logging.py --- lega/utils/logging.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lega/utils/logging.py b/lega/utils/logging.py index 2265403e..6007da29 100644 --- a/lega/utils/logging.py +++ b/lega/utils/logging.py @@ -3,7 +3,7 @@ Logs Formatter """ from logging import Formatter -from logging.handlers import SocketHandler as handler # or DatagramHandler ? +from logging.handlers import SocketHandler as handler # or DatagramHandler ? import json import re @@ -56,4 +56,4 @@ def format(self, record): if record.stack_info: log_record['stack_info'] = self.formatStack(record.stack_info) - return json.dumps(log_record) #, ensure_ascii=False) + return json.dumps(log_record) From c0bbfdb790e3b2b2347660dcba212887c247738e Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 12:47:24 +0200 Subject: [PATCH 15/25] Flake8 correction of utils/storage.py --- lega/utils/storage.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/lega/utils/storage.py b/lega/utils/storage.py index 5c651e32..db7871e6 100644 --- a/lega/utils/storage.py +++ b/lega/utils/storage.py @@ -23,7 +23,7 @@ def __init__(self): self.vault_area = Path(CONF.get_value('vault', 'location')) def location(self, file_id): - name = f"{file_id:0>20}" # filling with zeros, and 20 characters wide + name = f"{file_id:0>20}" # filling with zeros, and 20 characters wide name_bits = [name[i:i+3] for i in range(0, len(name), 3)] target = self.vault_area.joinpath(*name_bits) target.parent.mkdir(parents=True, exist_ok=True) @@ -35,7 +35,7 @@ def copy(self, fileobj, location): return os.stat(location).st_size @contextmanager - def open(self, path, mode = 'rb'): + def open(self, path, mode='rb'): f = open(path, mode) yield f f.close() @@ -45,9 +45,9 @@ class S3FileReader(object): """ Implements a few of the BufferedIOBase methods """ - def __init__(self, s3, bucket, path, mode='rb', blocksize = 1<<22): # 1<<22 = 4194304 = 4MB - - if mode != 'rb': # if mode not in ('rb', 'wb', 'ab'): + def __init__(self, s3, bucket, path, mode='rb', blocksize=1 << 22): # 1<<22 = 4194304 = 4MB + + if mode != 'rb': # if mode not in ('rb', 'wb', 'ab'): raise NotImplementedError(f"File mode '{mode}' not supported") self.mode = mode self.path = path @@ -70,11 +70,11 @@ def tell(self): return self.loc def seek(self, loc, whence=0): - if whence == 0: # from start + if whence == 0: # from start nloc = loc - elif whence == 1: # from here + elif whence == 1: # from here nloc = self.loc + loc - elif whence == 2: # from end + elif whence == 2: # from end nloc = self.size + loc else: raise ValueError("invalid whence (%s, should be 0, 1 or 2)" % whence) @@ -114,13 +114,13 @@ def read(self, length=-1): if self.closed: raise ValueError('I/O operation on closed file.') - if self.loc == self.size: # at the end already + if self.loc == self.size: # at the end already return b'' - if length < 0: # the rest of the file - length = self.size - self.loc + if length < 0: # the rest of the file + length = self.size - self.loc - end = min(self.loc + length, self.size) # in case it's too much + end = min(self.loc + length, self.size) # in case it's too much out = self._fetch(self.loc, end) self.loc += len(out) return out @@ -165,7 +165,7 @@ def _fetch(self, start, end, max_attempts=10): # if end > self.size: # end = self.size assert end <= self.size - #LOG.debug("Fetch: Bucket: %s, File=%s, Range: %s-%s, Chunk: %s", self.bucket, self.path, start, end, end-start) + # LOG.debug("Fetch: Bucket: %s, File=%s, Range: %s-%s, Chunk: %s", self.bucket, self.path, start, end, end-start) for i in range(max_attempts): try: resp = self.s3.get_object(Bucket=self.bucket, Key=self.path, Range='bytes=%i-%i' % (start, end - 1)) @@ -191,7 +191,6 @@ class S3Storage(): def __init__(self): import boto3 - import socket endpoint = CONF.get_value('vault', 'url') region = CONF.get_value('vault', 'region') @@ -205,9 +204,9 @@ def __init__(self): region_name=region, use_ssl=False, verify=False, - aws_access_key_id = access_key, - aws_secret_access_key = secret_key) - #LOG.debug(f'S3 client: {self.s3!r}') + aws_access_key_id=access_key, + aws_secret_access_key=secret_key) + # LOG.debug(f'S3 client: {self.s3!r}') try: LOG.debug('Creating "%s" bucket', bucket) self.bucket = bucket @@ -225,8 +224,7 @@ def copy(self, fileobj, location): return resp['ContentLength'] @contextmanager - def open(self, path, mode = 'rb'): + def open(self, path, mode='rb'): f = S3FileReader(self.s3, self.bucket, path, mode=mode) yield f f.close() - From 7d35c7bd3826dfc253dfc0cf3160ac07d7ed1a92 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 14:01:43 +0200 Subject: [PATCH 16/25] Flake8 correction of utils/db.py --- lega/utils/db.py | 71 ++++++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/lega/utils/db.py b/lega/utils/db.py index f60686ec..26e74210 100644 --- a/lega/utils/db.py +++ b/lega/utils/db.py @@ -21,16 +21,16 @@ LOG = logging.getLogger(__name__) ###################################### -## DB connection ## +# DB connection # ###################################### def fetch_args(d): """Initializing a connection to db.""" - db_args = { 'user' : d.get_value('postgres', 'user'), - 'password' : d.get_value('postgres', 'password'), - 'database' : d.get_value('postgres', 'db'), - 'host' : d.get_value('postgres', 'host'), - 'port' : d.get_value('postgres', 'port', conv=int) - } + db_args = {'user': d.get_value('postgres', 'user'), + 'password': d.get_value('postgres', 'password'), + 'database': d.get_value('postgres', 'db'), + 'host': d.get_value('postgres', 'host'), + 'port': d.get_value('postgres', 'port', conv=int) + } LOG.info(f"Initializing a connection to: {db_args['host']}:{db_args['port']}/{db_args['database']}") return db_args @@ -49,11 +49,11 @@ def wrapper(*args, **kwargs): backoff = try_interval while count < nb_try: try: - return func(*args,**kwargs) + return func(*args, **kwargs) except exception as e: LOG.debug(f"Database connection error: {e!r}") LOG.debug(f"Retrying in {backoff} seconds") - sleep( backoff ) + sleep(backoff) count += 1 backoff = (2 ** (count // 10)) * try_interval # from 0 to 9, sleep 1 * try_interval secs @@ -75,8 +75,9 @@ def _do_exit(): LOG.error("Could not connect to the database: Exiting") sys.exit(1) + ###################################### -## "Classic" code ## +# "Classic" code # ###################################### _conn = None def cache_connection(func): @@ -107,9 +108,9 @@ def insert_file(filename, user_id): with connect() as conn: with conn.cursor() as cur: cur.execute('SELECT insert_file(%(filename)s,%(user_id)s);', - { 'filename': filename, - 'user_id': user_id, - }) + {'filename': filename, + 'user_id': user_id, + }) file_id = (cur.fetchone())[0] if file_id: LOG.debug(f'Created id {file_id} for {filename}') @@ -135,14 +136,14 @@ def set_error(file_id, error, from_user=False): with connect() as conn: with conn.cursor() as cur: cur.execute('SELECT insert_error(%(file_id)s,%(h)s,%(etype)s,%(msg)s,%(from_user)s);', - {'h':hostname, 'etype': error.__class__.__name__, 'msg': repr(error), 'file_id': file_id, 'from_user': from_user}) + {'h': hostname, 'etype': error.__class__.__name__, 'msg': repr(error), 'file_id': file_id, 'from_user': from_user}) def get_info(file_id): """Retrieve information for ``file_id``.""" with connect() as conn: with conn.cursor() as cur: query = 'SELECT inbox_path, vault_path, stable_id, header from files WHERE id = %(file_id)s;' - cur.execute(query, { 'file_id': file_id}) + cur.execute(query, {'file_id': file_id}) return cur.fetchone() def _set_status(file_id, status): @@ -153,7 +154,7 @@ def _set_status(file_id, status): with conn.cursor() as cur: cur.execute('UPDATE files SET status = %(status)s WHERE id = %(file_id)s;', {'status': status, - 'file_id': file_id }) + 'file_id': file_id}) def mark_in_progress(file_id): return _set_status(file_id, 'In progress') @@ -173,7 +174,7 @@ def set_stable_id(file_id, stable_id): 'WHERE id = %(file_id)s;', {'status': 'Ready', 'file_id': file_id, - 'stable_id': stable_id }) + 'stable_id': stable_id}) def store_header(file_id, header): @@ -187,8 +188,7 @@ def store_header(file_id, header): 'SET header = %(header)s ' 'WHERE id = %(file_id)s;', {'file_id': file_id, - 'header': header, - }) + 'header': header}) def set_archived(file_id, vault_path, vault_filesize): @@ -206,48 +206,48 @@ def set_archived(file_id, vault_path, vault_filesize): {'status': 'Archived', 'file_id': file_id, 'vault_path': vault_path, - 'vault_filesize': vault_filesize - }) + 'vault_filesize': vault_filesize}) + ###################################### -## Decorator ## +# Decorator # ###################################### _channel = None -def catch_error(func): + +def catch_error(func): # noqa: C901 '''Decorator to store the raised exception in the database''' @wraps(func) def wrapper(*args): try: return func(*args) except Exception as e: - if isinstance(e,AssertionError): + if isinstance(e, AssertionError): raise e exc_type, _, exc_tb = sys.exc_info() g = traceback.walk_tb(exc_tb) - frame, lineno = next(g) # that should be the decorator + frame, lineno = next(g) # that should be the decorator try: - frame, lineno = next(g) # that should be where is happened + frame, lineno = next(g) # that should be where is happened except StopIteration: - pass # In case the trace is too short + pass # In case the trace is too short - #fname = os.path.split(frame.f_code.co_filename)[1] fname = frame.f_code.co_filename LOG.error(f'Exception: {exc_type} in {fname} on line: {lineno}') - from_user = isinstance(e,FromUser) + from_user = isinstance(e, FromUser) cause = e.__cause__ or e - LOG.error(f'{cause!r} (from user: {from_user})') # repr = Technical + LOG.error(f'{cause!r} (from user: {from_user})') # repr = Technical try: - data = args[-1] # data is the last argument - file_id = data.get('file_id', None) # should be there + data = args[-1] # data is the last argument + file_id = data.get('file_id', None) # should be there if file_id: set_error(file_id, cause, from_user) LOG.debug('Catching error on file id: %s', file_id) - if from_user: # Send to CentralEGA - org_msg = data.pop('org_msg', None) # should be there - org_msg['reason'] = str(cause) # str = Informal + if from_user: # Send to CentralEGA + org_msg = data.pop('org_msg', None) # should be there + org_msg['reason'] = str(cause) # str = Informal LOG.info(f'Sending user error to local broker: {org_msg}') global _channel if _channel is None: @@ -273,6 +273,7 @@ def wrapper(*args): raise return wrapper + # Testing connection with `python -m lega.utils.db` if __name__ == '__main__': CONF.setup(sys.argv) From 074aed5ecb42da6c746879b94b9ca36284485c12 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 13:05:15 +0200 Subject: [PATCH 17/25] Updated tests to work with flake8-ified verify.py --- tests/test_verify.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_verify.py b/tests/test_verify.py index 97dce5a6..d176382e 100644 --- a/tests/test_verify.py +++ b/tests/test_verify.py @@ -117,12 +117,10 @@ def test_main(self, mock_consume, mock_connection): @tempdir() @mock.patch('lega.verify.db') @mock.patch('lega.verify.body_decrypt') - @mock.patch('lega.verify.publish') @mock.patch('lega.verify.get_records') - def test_work(self, mock_records, mock_publish, mock_decrypt, mock_db, filedir): + def test_work(self, mock_records, mock_decrypt, mock_db, filedir): """Test verify worker, should send a messge.""" # Mocking a lot of stuff, ast it is previously tested - mock_publish.return_value = mock.MagicMock() mock_db.status.return_value = mock.Mock() mock_records.return_value = ['data'], 'key_id' mock_decrypt.return_value = mock.Mock() From 4d5c5e9074afe0caaa43d73a33f60cbe3639ba6e Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 13:54:46 +0200 Subject: [PATCH 18/25] Flake8 correction of tests/ subdirectory --- tests/conftest.py | 4 ++-- tests/pgp_data.py | 1 + tests/test_basic_functions.py | 9 ++++----- tests/test_ingest.py | 3 +-- tests/test_keyserver.py | 9 ++++++++- tests/test_storage.py | 1 - 6 files changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index cd9118c1..c9fa47a6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,7 +4,7 @@ def pytest_itemcollected(item): par = item.parent.obj node = item.obj # First line only - prefix = par.__doc__.split('\n',1)[0].strip() if par.__doc__ else par.__class__.__name__ - suffix = node.__doc__.split('\n',1)[0].strip() if node.__doc__ else node.__name__ + prefix = par.__doc__.split('\n', 1)[0].strip() if par.__doc__ else par.__class__.__name__ + suffix = node.__doc__.split('\n', 1)[0].strip() if node.__doc__ else node.__name__ if prefix or suffix: item._nodeid = ' | '.join((prefix, suffix)) diff --git a/tests/pgp_data.py b/tests/pgp_data.py index 80cd6ba3..27f351c2 100644 --- a/tests/pgp_data.py +++ b/tests/pgp_data.py @@ -1,3 +1,4 @@ +# flake8: noqa PGP_PASSPHRASE = 'crazywow' PGP_NAME = 'PyTest' PGP_COMMENT = 'Fake Key for PyTest' diff --git a/tests/test_basic_functions.py b/tests/test_basic_functions.py index 20a04bff..7bbecc62 100644 --- a/tests/test_basic_functions.py +++ b/tests/test_basic_functions.py @@ -9,7 +9,6 @@ import io from testfixtures import tempdir from . import pgp_data -import sys from io import StringIO @@ -94,12 +93,12 @@ def test_supported_algorithms(self): def test_config_main(self): """Testing main configuration.""" with mock.patch('sys.stdout', new=StringIO()) as fake_stdout: - main(['--conf', 'fake/conf.ini']) - self.assertTrue(fake_stdout.getvalue(), 'Configuration files:') + main(['--conf', 'fake/conf.ini']) + self.assertTrue(fake_stdout.getvalue(), 'Configuration files:') with mock.patch('sys.stdout', new=StringIO()) as fake_stdout: - main(['--list']) - self.assertTrue(fake_stdout.getvalue(), 'Configuration values:') + main(['--list']) + self.assertTrue(fake_stdout.getvalue(), 'Configuration values:') def test_do_exit(self): """Testing simple exit.""" diff --git a/tests/test_ingest.py b/tests/test_ingest.py index baf54d73..ff964666 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -1,6 +1,5 @@ import unittest from lega.ingest import main, work -from lega.utils.exceptions import NotFoundInInbox from unittest import mock from testfixtures import tempdir from pathlib import PosixPath @@ -36,7 +35,7 @@ def test_work(self, mock_db, mock_path, mock_header, filedir): store.open.return_value = mock.MagicMock() mock_broker = mock.MagicMock(name='channel') mock_broker.channel.return_value = mock.Mock() - infile = filedir.write('infile.in', bytearray.fromhex(pgp_data.ENC_FILE)) + infile = filedir.write('infile.in', bytearray.fromhex(pgp_data.ENC_FILE)) data = {'filepath': infile, 'user': 'user_id@elixir-europe.org'} result = work(store, mock_broker, data) mocked = {'filepath': infile, 'user': 'user_id@elixir-europe.org', diff --git a/tests/test_keyserver.py b/tests/test_keyserver.py index 53de74b0..d41f7e15 100644 --- a/tests/test_keyserver.py +++ b/tests/test_keyserver.py @@ -142,11 +142,18 @@ def test_check_ttl(self): end_date = date_1 + datetime.timedelta(days=10) with self._key.unlock(pgp_data.PGP_PASSPHRASE) as privkey: self._cache.set('test_key', privkey, end_date.strftime('%d/%b/%y %H:%M:%S')) + today = datetime.datetime.today().strftime(self.FMT) tdelta = end_date - datetime.datetime.strptime(today, self.FMT) tdelta = datetime.timedelta(days=tdelta.days, seconds=tdelta.seconds) + + days = tdelta.days + hours = tdelta.days * 24 + tdelta.seconds // 3600 + minutes = tdelta.seconds % 3600 // 60 + seconds = tdelta.seconds + expected_value = [{"keyID": "test_key", - "ttl": f"{tdelta.days} days {tdelta.days * 24 + tdelta.seconds // 3600} hours {(tdelta.seconds % 3600) // 60} minutes {tdelta.seconds} seconds"}] + "ttl": f"{days} days {hours} hours {minutes} minutes {seconds} seconds"}] self.assertEqual(self._cache.check_ttl(), expected_value) self._cache.clear() diff --git a/tests/test_storage.py b/tests/test_storage.py index c9271533..3967cc75 100644 --- a/tests/test_storage.py +++ b/tests/test_storage.py @@ -6,7 +6,6 @@ from io import UnsupportedOperation, BufferedReader from unittest import mock import boto3 -import botocore.response as br class TestFileStorage(unittest.TestCase): From 8c1c79e0be6dee5d9a59d8745b4bf6ef59ad1e74 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 14:01:16 +0200 Subject: [PATCH 19/25] Updated tox configuration to run flake8 --- tox.ini | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tox.ini b/tox.ini index 05347f88..9516b5cd 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = unit_tests, flake8 +envlist = unit_tests, flake8, flake8-tests [flake8] ignore = E226,E302 @@ -9,8 +9,12 @@ max-complexity = 10 [testenv:flake8] changedir = lega deps = flake8 -# Do not run this yet, decide if we are going to use style guide enforcement -# commands = flake8 . +commands = flake8 . + +[testenv:flake8-tests] +changedir = tests +deps = flake8 +commands = flake8 . [testenv:unit_tests] passenv = TRAVIS TRAVIS_* From ccf39ea46b2bcbea88b5083a86b1f5bd73c5f177 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 14:23:24 +0200 Subject: [PATCH 20/25] Run flake8 in travis --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 9516b5cd..2d019629 100644 --- a/tox.ini +++ b/tox.ini @@ -28,4 +28,4 @@ commands = pytest -x --cov=lega tests/ [travis] unignore_outcomes = True python = - 3.6: unit_tests + 3.6: unit_tests, flake8, flake8-tests From ea41402bcf0ba65ec2ad9cb4d36e1a35af13904c Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 14:49:03 +0200 Subject: [PATCH 21/25] Streamlined flake8 config for tox --- tox.ini | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tox.ini b/tox.ini index 2d019629..b8d918f1 100644 --- a/tox.ini +++ b/tox.ini @@ -1,18 +1,16 @@ [tox] -envlist = unit_tests, flake8, flake8-tests +envlist = unit_tests, flake8 [flake8] ignore = E226,E302 +exclude = + docker, + extras, + .tox max-line-length = 160 max-complexity = 10 [testenv:flake8] -changedir = lega -deps = flake8 -commands = flake8 . - -[testenv:flake8-tests] -changedir = tests deps = flake8 commands = flake8 . @@ -28,4 +26,4 @@ commands = pytest -x --cov=lega tests/ [travis] unignore_outcomes = True python = - 3.6: unit_tests, flake8, flake8-tests + 3.6: unit_tests, flake8 From 386a571ddd9a36c188cd31a2b24371c7fb38f915 Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 14:49:57 +0200 Subject: [PATCH 22/25] Flake8 correction of setup.py --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 24d82320..8a8ddf7c 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ ''', packages=['lega', 'lega/utils', 'lega/conf'], include_package_data=False, - package_data={ 'lega': ['conf/loggers/*.yaml', 'conf/defaults.ini'] }, + package_data={'lega': ['conf/loggers/*.yaml', 'conf/defaults.ini']}, zip_safe=False, entry_points={ 'console_scripts': [ @@ -30,5 +30,5 @@ 'ega-conf = lega.conf.__main__:main', ] }, - platforms = 'any', -) + platforms='any', + ) From 04b74128c686a70b42589915604556433380ccdf Mon Sep 17 00:00:00 2001 From: Johan Viklund Date: Wed, 12 Sep 2018 14:52:22 +0200 Subject: [PATCH 23/25] Flake8 correction of docs/setup.py --- docs/conf.py | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 0fd0041a..f15116a9 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,11 +7,9 @@ from unittest.mock import MagicMock # Get the project root dir, which is the parent dir of this -#sys.path.insert(0, os.path.dirname(os.getcwd())) sys.path.insert(0, os.path.abspath('..')) -#print('PYTHONPATH =', sys.path) -import lega +import lega # noqa: E402 # -- General configuration ------------------------------------------------ @@ -39,7 +37,7 @@ def __getattr__(cls, name): 'sphinx.ext.viewcode', 'sphinx.ext.githubpages', 'sphinx.ext.todo', -] + ] # Add any paths that contain templates here, relative to this directory. templates_path = ['templates'] @@ -97,21 +95,11 @@ def __getattr__(cls, name): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -# -#html_theme = 'alabaster' -# html_theme_options = { -# 'fixed_sidebar': True, -# 'show_powered_by': False, -# #'badge_branch': 'dev', -# 'github_repo': 'https://github.com/NBISweden/LocalEGA', -# 'github_button': True, -# } html_theme = 'sphinx_rtd_theme' html_theme_options = { 'collapse_navigation': True, 'sticky_navigation': True, - #'navigation_depth': 4, 'display_version': True, 'prev_next_buttons_location': 'bottom', } @@ -127,13 +115,7 @@ def __getattr__(cls, name): # This is required for the alabaster theme # refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars html_sidebars = { - '**': [ - #'about.html', - #'navigation.html', - #'relations.html', # needs 'show_related': True theme option to display - #'searchbox.html', - #'donate.html', - ] + '**': [] } @@ -142,9 +124,10 @@ def __getattr__(cls, name): def setup(app): app.add_stylesheet('custom.css') + # -- Other stuff ---------------------------------------------------------- htmlhelp_basename = 'LocalEGA' latex_elements = {} -latex_documents = [ (master_doc, 'LocalEGA.tex', 'Local EGA', 'NBIS System Developers', 'manual') ] -man_pages = [ (master_doc, 'localega', 'Local EGA', [author], 1) ] -texinfo_documents = [ (master_doc, 'LocalEGA', 'Local EGA', author, 'LocalEGA', 'Local extension to the European Genomic Archive', 'Miscellaneous') ] +latex_documents = [(master_doc, 'LocalEGA.tex', 'Local EGA', 'NBIS System Developers', 'manual')] +man_pages = [(master_doc, 'localega', 'Local EGA', [author], 1)] +texinfo_documents = [(master_doc, 'LocalEGA', 'Local EGA', author, 'LocalEGA', 'Local extension to the European Genomic Archive', 'Miscellaneous')] From 152dcda7679b027b7adc3fda88438971a97c1934 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Wed, 12 Sep 2018 16:24:52 +0300 Subject: [PATCH 24/25] small update to docs for testing. --- docs/tests.rst | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/docs/tests.rst b/docs/tests.rst index c21e2615..ba93e22c 100644 --- a/docs/tests.rst +++ b/docs/tests.rst @@ -15,7 +15,22 @@ Unit Tests Unit tests are minimal: Given a set of input values for a chosen function, they execute the function and check if the output has the expected values. Moreover, they capture triggered exceptions and -errors. Unit tests can be run using the ``tox`` commands. +errors. Additionally we also perform pep8 and pep257 style guide checks +using `flake8 `_ with the following +configuration: + +.. code-block:: yaml + + [flake8] + ignore = E226,E302 + exclude = + docker, + extras, + .tox + max-line-length = 160 + max-complexity = 10 + +Unit tests can be run using the ``tox`` commands. .. code-block:: console From 9f5cf95db69a2d08167831225ff1d778c582d8d3 Mon Sep 17 00:00:00 2001 From: Stefan Negru Date: Thu, 13 Sep 2018 07:58:25 +0300 Subject: [PATCH 25/25] remove lega.qc from plus logging change --- docs/code.rst | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/docs/code.rst b/docs/code.rst index f29c31fb..a4a1784e 100644 --- a/docs/code.rst +++ b/docs/code.rst @@ -35,14 +35,6 @@ Ingestion Worker :members: -.. - *************** - Quality Control - *************** - - .. automodule:: lega.qc - :members: - ********* Keyserver ********* @@ -64,7 +56,8 @@ Utility Functions lega.utils.storage lega.utils.eureka lega.utils.exceptions - lega.utils.logging + lega.utils.logging.LEGAHandler + lega.utils.logging.JSONFormatter