Merge pull request #97 from EGA-archive/requirements

Requirements and refactoring of cega-users

.dockerignore

@@ -1,15 +1,10 @@
-.coverage
-.coveragerc
-.DS_Store
-.git
-.github
-.gitignore
-.tox
+.*
 CONTRIBUTING.md
 deploy
+!deploy/requirements.txt
 Dockerfile
 docs
-lega.egg-info
+lega.*
 LICENSE
 README.md
 readthedocs.yml

.github/workflows/testsuite.yml

@@ -38,6 +38,7 @@ jobs:
         sudo ./install.sh /usr/local
         popd
         pip install -r deploy/bootstrap/requirements.txt
+        pip install -r tests/requirements.txt
         sudo apt-get install expect
     - name: Bootstrap (options ${{ matrix.bootstrap }})
       run: |

Dockerfile

@@ -3,9 +3,10 @@
 ##########################
 FROM python:3.8-alpine3.11 as BUILD
 
-RUN apk add git postgresql-dev gcc musl-dev libffi-dev make gnupg
+RUN apk add git libressl-dev postgresql-dev gcc musl-dev libffi-dev make
 
-COPY requirements.txt /root/LocalEGA/requirements.txt
+# This will pin the package versions
+COPY deploy/requirements.txt /root/LocalEGA/requirements.txt
 RUN pip install --upgrade pip && \
     pip install -r /root/LocalEGA/requirements.txt
 

deploy/README.md

@@ -6,16 +6,17 @@ You can then [generate the private data](bootstrap), with:
 
 	make -C bootstrap
 
-This requires `openssl` (>=1.1), `ssh-keygen` (=>6.5).
+This requires `openssl` (>=1.1), `ssh-keygen` (=>6.5).  
+Install the required python packages with `pip install -r bootstrap/requirements.txt`.
 
-The command will create a `.env` file and a `private` folder holding
-the necessary data (ie the master keypair, the TLS
+Bootstrapping will create a `.env` file and a `private` directory
+holding the necessary data (ie the master keypair, the TLS
 certificates for internal communication, passwords, default users,
-etc...)
+configuration files, etc...)
 
 The passwords are in `private/secrets` and the errors (if any) are in `private/.err`.
 
-The credentials for the test users (john, jane and dummy) are found in `private/users`.
+The credentials for the test users are found in `private/users`.
 
 ## Convenient commands
 
@@ -63,3 +64,7 @@ We use 2 stubbing services in order to fake the necessary Central EGA components
 |-------------:|------|
 | `cega-users` | Sets up a small list of test users |
 | `cega-mq`    | Sets up a RabbitMQ message broker with appropriate accounts, exchanges, queues and bindings |
+
+If the `cega-users` is not built, it will be build by docker-compose. If you want to build yourself, you can run:
+
+	docker-compose build cega-users

deploy/bootstrap/cega-users/Dockerfile

@@ -0,0 +1,26 @@
+##########################
+## Build env
+##########################
+FROM python:3.8-alpine3.11 as BUILD
+
+RUN apk add gcc musl-dev
+
+RUN pip install --upgrade pip && \
+    pip install aiohttp
+
+##########################
+## Final image
+##########################
+FROM python:3.8-alpine3.11
+
+LABEL maintainer "EGA System Developers"
+LABEL org.label-schema.schema-version="1.0"
+LABEL org.label-schema.vcs-url="https://github.com/EGA-archive/LocalEGA"
+
+COPY --from=BUILD usr/local/lib/python3.8/ usr/local/lib/python3.8/
+
+VOLUME /cega/users
+
+COPY server.py /cega/server.py
+
+ENTRYPOINT ["python", "/cega/server.py", "0.0.0.0", "443", "/cega/users"]

deploy/bootstrap/requirements.txt

@@ -1,6 +1,7 @@
-PyYaml
-pika==1.1.0
-bcrypt
-ruamel.yaml
-docopt
-git+https://github.com/EGA-archive/crypt4gh.git
+bcrypt==3.1.7
+cffi==1.13.2
+docopt==0.6.2
+pycparser==2.19
+PyYAML==5.3
+ruamel.yaml==0.16.6
+six==1.14.0

deploy/bootstrap/run/cega/services.py

@@ -13,19 +13,17 @@
     'services': {
         'cega-users': {
             'hostname': f'cega-users{HOSTNAME_DOMAIN}',
-            'ports': ["15671:443"],
-            'image': 'egarchive/lega-base:latest',
+            # 'ports': ["15671:443"],
+            'build': '../bootstrap/cega-users',
+            'image': 'egarchive/cega-users:latest',
             'container_name': f'cega-users{HOSTNAME_DOMAIN}',
             'volumes': [
-                '../bootstrap/cega-users.py:/cega/users.py',
                 './users:/cega/users',
                 './certs/cega-users.cert.pem:/cega/ssl.crt',
                 './certs/cega-users.sec.pem:/cega/ssl.key',
                 './certs/CA.cega-users.cert.pem:/cega/CA.crt',
             ],
             'networks': ['external'],
-            'user': 'root',
-            'entrypoint': ["python", "/cega/users.py", "0.0.0.0", "443", "/cega/users"],
         },
 
         'cega-mq': {

deploy/bootstrap/run/lega/services.py

@@ -148,6 +148,9 @@ def main(cega_conf, conf, args):
                 'S3_SECRET_KEY='+conf.get('s3','secret_key'),
             ] if with_s3 else []),
             'hostname': f'ingest{HOSTNAME_DOMAIN}',
+            'build': '../..',  # Just in case we docker-compose up before building the image locally
+                               # This might be useless since the image from the master branch is built on docker hub.
+                               # so it will get downloaded
             'image': 'egarchive/lega-base:latest',
             'container_name': f'ingest{HOSTNAME_DOMAIN}',
             'volumes': [

deploy/requirements.txt

@@ -0,0 +1,18 @@
+bcrypt==3.1.7
+boto3==1.11.7
+botocore==1.14.7
+cffi==1.13.2
+cryptography==2.8
+docopt==0.6.2
+docutils==0.15.2
+jmespath==0.9.4
+pika==1.1.0
+psycopg2==2.8.4
+pycparser==2.19
+PyNaCl==1.3.0
+python-dateutil==2.8.1
+PyYAML==5.3
+s3transfer==0.3.1
+six==1.14.0
+urllib3==1.25.8
+git+https://github.com/EGA-archive/crypt4gh.git

docs/requirements.txt

@@ -0,0 +1,29 @@
+# That's just "pip install Sphinx PyYAML sphinx_rtd_theme recommonmark"
+alabaster==0.7.12
+Babel==2.8.0
+certifi==2019.11.28
+chardet==3.0.4
+commonmark==0.9.1
+docutils==0.16
+idna==2.8
+imagesize==1.2.0
+Jinja2==2.10.3
+MarkupSafe==1.1.1
+packaging==20.0
+Pygments==2.5.2
+pyparsing==2.4.6
+pytz==2019.3
+PyYAML==5.3
+recommonmark==0.6.0
+requests==2.22.0
+six==1.14.0
+snowballstemmer==2.0.0
+Sphinx==2.3.1
+sphinx-rtd-theme==0.4.3
+sphinxcontrib-applehelp==1.0.1
+sphinxcontrib-devhelp==1.0.1
+sphinxcontrib-htmlhelp==1.0.2
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.2
+sphinxcontrib-serializinghtml==1.1.3
+urllib3==1.25.8

lega/conf/__init__.py

@@ -16,7 +16,6 @@
 import logging
 logging.setLoggerClass(lega_logging.LEGALogger)
 
-import sys
 import os
 import configparser
 import warnings
@@ -29,6 +28,7 @@
 CONF_FILE = os.getenv('LEGA_CONF', '/etc/ega/conf.ini')
 LOG = logging.getLogger(__name__)
 
+
 def get_from_file(filepath, mode='rb', remove_after=False):
     """Return file content.
 
@@ -37,31 +37,32 @@ def get_from_file(filepath, mode='rb', remove_after=False):
     try:
         with open(filepath, mode) as s:
             return s.read()
-    except: # Crash if not found, or permission denied
-        raise ValueError(f'Error loading {filepath}')
+    except Exception as e:  # Crash if not found, or permission denied
+        raise ValueError(f'Error loading {filepath}') from e
     finally:
         if remove_after:
             try:
                 os.remove(filepath)
-            except: # Crash if not found, or permission denied
-                LOG.warning('Could not remove %s', filepath)
+            except Exception:  # Crash if not found, or permission denied
+                LOG.warning('Could not remove %s', filepath, exc_info=True)
+
 
 def convert_sensitive(value):
     """Fetch a sensitive value from different sources.
 
     * If `value` starts with 'env://', we strip it out and the remainder acts as the name of an environment variable to read.
     If the environment variable does not exist, we raise a ValueError exception.
-    
+
     * If `value` starts with 'file://', we strip it out and the remainder acts as the filepath of a file to read (in text mode).
     If any error occurs while read the file content, we raise a ValueError exception.
 
     * If `value` starts with 'secret://', we strip it out and the remainder acts as the filepath of a file to read (in binary mode), and we remove it after.
     If any error occurs while read the file content, we raise a ValueError exception.
-    
+
     * If `value` starts with 'value://', we strip it out and the remainder acts as the value itself.
     It is used to enforce the value, in case its content starts with env:// or file:// (eg a file:// URL).
-    
-    * Otherwise, `value` is the value content itself. 
+
+    * Otherwise, `value` is the value content itself.
     """
     if value is None:  # Not found
         return None
@@ -85,7 +86,7 @@ def convert_sensitive(value):
         return envvalue
 
     if value.startswith('file://'):
-        path=value[7:]
+        path = value[7:]
         LOG.debug('Loading value from path: %s', path)
         statinfo = os.stat(path)
         if statinfo.st_mode & stat.S_IRGRP or statinfo.st_mode & stat.S_IROTH:
@@ -98,7 +99,7 @@ def convert_sensitive(value):
         return get_from_file(path, mode='rt')  # str
 
     if value.startswith('secret://'):
-        path=value[9:]
+        path = value[9:]
         LOG.debug('Loading secret from path: %s', path)
         return get_from_file(path, mode='rb', remove_after=True)  # bytes
 
@@ -113,11 +114,7 @@ class Configuration(configparser.RawConfigParser):
 
     def __init__(self):
         """Set up."""
-        if not CONF_FILE:
-            print('No configuration found', file=sys.stderr)
-            print('Bailing out', file=sys.stderr)
-            sys.exit(2)
-
+        # Load the configuration settings
         configparser.RawConfigParser.__init__(self,
                                               delimiters=('=', ':'),
                                               comment_prefixes=('#', ';'),
@@ -126,28 +123,40 @@ def __init__(self):
                                               converters={
                                                   'sensitive': convert_sensitive,
                                               })
-        self.read([CONF_FILE], encoding='utf-8')
-        try:
-            self.load_log(LOG_FILE)
-        except Exception as e:
-            # import traceback
-            # traceback.print_stack()
-            print(f'No logging supplied: {e!r}', file=sys.stderr)
-            if e.__cause__:
-                print('Cause: {e.__cause__!r}', file=sys.stderr)
-
+        if (
+                not CONF_FILE  # has no value
+                or
+                not os.path.isfile(CONF_FILE)  # does not exist
+                or
+                not os.access(CONF_FILE, os.R_OK)  # is not readable
+        ):
+            warnings.warn("No configuration settings found", UserWarning, stacklevel=2)
+        else:
+            self.read([CONF_FILE], encoding='utf-8')
+
+        # Configure the logging system
+        if not LOG_FILE:
+            warnings.warn("No logging supplied", UserWarning, stacklevel=2)
+        else:
+            try:
+                self._load_log(LOG_FILE)
+            except Exception as e:
+                # import traceback
+                # traceback.print_stack()
+                warnings.warn(f"No logging supplied: {e!r}", UserWarning, stacklevel=3)
+                if e.__cause__:
+                    warnings.warn(f'Cause: {e.__cause__!r}', UserWarning, stacklevel=3)
+
     def __repr__(self):
         """Show the configuration files."""
         res = f'Configuration file: {CONF_FILE}'
         if self.logger:
             res += f'\nLogging settings loaded from {self.logger}'
         return res
 
-    def load_log(self, filename):
+    def _load_log(self, filename):
         """Try to load `filename` as configuration file for logging."""
-        if not filename:
-            raise ValueError('No logging supplied')
-
+        assert(filename)
         _here = Path(__file__).parent
 
         # Try first if it is a default logger

lega/conf/__main__.py

@@ -1,20 +1,13 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 
-import sys
-
-from . import CONF
-
+if __name__ == "__main__":
 
-def main(print_values=False):
-    """Run the main routine, for loading configuration."""
+    import sys
+    from . import CONF
 
     print(repr(CONF))
 
-    if print_values:
+    if len(sys.argv) > 1 and sys.argv[1] == 'list':
         print("\nConfiguration values:")
         CONF.write(sys.stdout)
-
-
-if __name__ == "__main__":
-    main(print_values=len(sys.argv) > 1 and sys.argv[1] == 'list')

lega/conf/logging.py

@@ -3,6 +3,7 @@
 
 from logging import Logger
 
+
 class _wrapper():
     """Wrapper class for the correlation id."""
 

lega/utils/__init__.py

@@ -3,10 +3,15 @@
 Used internally.
 """
 
+
 def sanitize_user_id(user):
     """Return username without host part of an ID on the form name@something."""
     return user.split('@')[0]
 
+
 def redact_url(url):
     """Remove user:password from the URL."""
-    return '{}[redacted]@{}'.format(url[:url.index('://')+3], url.split('@',1)[-1])
+    protocol = url[:url.index('://')+3]
+    remainder = url.split('@', 1)[-1]
+    # return f'{protocol}[redacted]@{remainder}'
+    return protocol + '[redacted]@' + remainder